2024-11-18 19:04:52,008 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-18 19:04:52,023 main DEBUG Took 0.012465 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 19:04:52,024 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 19:04:52,024 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 19:04:52,025 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 19:04:52,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,039 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 19:04:52,056 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,058 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,059 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,059 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,060 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,060 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,061 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,062 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,062 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,063 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,064 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,064 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,065 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,065 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,066 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,067 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,067 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,068 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,069 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,070 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 19:04:52,071 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,072 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 19:04:52,074 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 19:04:52,075 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 19:04:52,078 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 19:04:52,078 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 19:04:52,080 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 19:04:52,081 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 19:04:52,093 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 19:04:52,097 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 19:04:52,100 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 19:04:52,100 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 19:04:52,101 main DEBUG createAppenders(={Console}) 2024-11-18 19:04:52,102 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-18 19:04:52,102 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-18 19:04:52,103 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-18 19:04:52,103 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 19:04:52,104 main DEBUG OutputStream closed 2024-11-18 19:04:52,104 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 19:04:52,105 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 19:04:52,105 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-18 19:04:52,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 19:04:52,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 19:04:52,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 19:04:52,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 19:04:52,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 19:04:52,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 19:04:52,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 19:04:52,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 19:04:52,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 19:04:52,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 19:04:52,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 19:04:52,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 19:04:52,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 19:04:52,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 19:04:52,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 19:04:52,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 19:04:52,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 19:04:52,221 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 19:04:52,225 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 19:04:52,225 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-18 19:04:52,226 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 19:04:52,227 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-18T19:04:52,285 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-18 19:04:52,289 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 19:04:52,290 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T19:04:52,557 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c 2024-11-18T19:04:52,561 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-18T19:04:52,563 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:04:52,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T19:04:52,577 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31, deleteOnExit=true 2024-11-18T19:04:52,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T19:04:52,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/test.cache.data in system properties and HBase conf 2024-11-18T19:04:52,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T19:04:52,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir in system properties and HBase conf 2024-11-18T19:04:52,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T19:04:52,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T19:04:52,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T19:04:52,696 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T19:04:52,834 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T19:04:52,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T19:04:52,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T19:04:52,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T19:04:52,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T19:04:52,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T19:04:52,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T19:04:52,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T19:04:52,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T19:04:52,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T19:04:52,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/nfs.dump.dir in system properties and HBase conf 2024-11-18T19:04:52,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/java.io.tmpdir in system properties and HBase conf 2024-11-18T19:04:52,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T19:04:52,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T19:04:52,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T19:04:53,985 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T19:04:54,066 INFO [Time-limited test {}] log.Log(170): Logging initialized @2883ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T19:04:54,151 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:04:54,217 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:04:54,243 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:04:54,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:04:54,246 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T19:04:54,264 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:04:54,267 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f045f3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:04:54,269 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b22dca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T19:04:54,492 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@38e78c7b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/java.io.tmpdir/jetty-localhost-33969-hadoop-hdfs-3_4_1-tests_jar-_-any-17616722209178246311/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T19:04:54,504 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c336369{HTTP/1.1, (http/1.1)}{localhost:33969} 2024-11-18T19:04:54,504 INFO [Time-limited test {}] server.Server(415): Started @3322ms 2024-11-18T19:04:54,911 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:04:54,919 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:04:54,920 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:04:54,921 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:04:54,921 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T19:04:54,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:04:54,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T19:04:55,049 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@cb44ad4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/java.io.tmpdir/jetty-localhost-46833-hadoop-hdfs-3_4_1-tests_jar-_-any-13280348368958461239/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T19:04:55,050 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:46833} 2024-11-18T19:04:55,050 INFO [Time-limited test {}] server.Server(415): Started @3868ms 2024-11-18T19:04:55,119 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T19:04:55,634 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/data/data2/current/BP-400906797-172.17.0.2-1731956693525/current, will proceed with Du for space computation calculation, 2024-11-18T19:04:55,634 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/data/data1/current/BP-400906797-172.17.0.2-1731956693525/current, will proceed with Du for space computation calculation, 2024-11-18T19:04:55,679 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T19:04:55,735 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xccc3599227052ef7 with lease ID 0x65a90fe1acb9f2ca: Processing first storage report for DS-0447fa01-1bd9-4334-a14d-506deb006e71 from datanode DatanodeRegistration(127.0.0.1:33673, datanodeUuid=e75a332a-38ee-4bff-a5be-9a275960cd0b, infoPort=41227, infoSecurePort=0, ipcPort=35119, storageInfo=lv=-57;cid=testClusterID;nsid=423244424;c=1731956693525) 2024-11-18T19:04:55,736 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xccc3599227052ef7 with lease ID 0x65a90fe1acb9f2ca: from storage DS-0447fa01-1bd9-4334-a14d-506deb006e71 node DatanodeRegistration(127.0.0.1:33673, datanodeUuid=e75a332a-38ee-4bff-a5be-9a275960cd0b, infoPort=41227, infoSecurePort=0, ipcPort=35119, storageInfo=lv=-57;cid=testClusterID;nsid=423244424;c=1731956693525), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T19:04:55,737 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xccc3599227052ef7 with lease ID 0x65a90fe1acb9f2ca: Processing first storage report for DS-9d4a80a4-80e8-489f-892b-3b81db7ea31d from datanode DatanodeRegistration(127.0.0.1:33673, datanodeUuid=e75a332a-38ee-4bff-a5be-9a275960cd0b, infoPort=41227, infoSecurePort=0, ipcPort=35119, storageInfo=lv=-57;cid=testClusterID;nsid=423244424;c=1731956693525) 2024-11-18T19:04:55,737 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xccc3599227052ef7 with lease ID 0x65a90fe1acb9f2ca: from storage DS-9d4a80a4-80e8-489f-892b-3b81db7ea31d node DatanodeRegistration(127.0.0.1:33673, datanodeUuid=e75a332a-38ee-4bff-a5be-9a275960cd0b, infoPort=41227, infoSecurePort=0, ipcPort=35119, storageInfo=lv=-57;cid=testClusterID;nsid=423244424;c=1731956693525), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T19:04:55,758 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c 2024-11-18T19:04:55,839 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/zookeeper_0, clientPort=64381, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T19:04:55,850 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64381 2024-11-18T19:04:55,865 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:04:55,869 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:04:56,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741825_1001 (size=7) 2024-11-18T19:04:56,506 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 with version=8 2024-11-18T19:04:56,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging 2024-11-18T19:04:56,518 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-18T19:04:56,518 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-18T19:04:56,519 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-18T19:04:56,519 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-18T19:04:56,609 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T19:04:56,878 INFO [Time-limited test {}] client.ConnectionUtils(128): master/818bff808382:0 server-side Connection retries=45 2024-11-18T19:04:56,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:04:56,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T19:04:56,894 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T19:04:56,894 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:04:56,894 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T19:04:57,040 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T19:04:57,107 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T19:04:57,120 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T19:04:57,126 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T19:04:57,153 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 11507 (auto-detected) 2024-11-18T19:04:57,155 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T19:04:57,173 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35649 2024-11-18T19:04:57,196 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35649 connecting to ZooKeeper ensemble=127.0.0.1:64381 2024-11-18T19:04:57,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356490x0, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T19:04:57,564 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35649-0x1000c059bf10000 connected 2024-11-18T19:04:57,593 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:04:57,595 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:04:57,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-18T19:04:57,628 INFO [Time-limited test {}] log.Log(170): Logging initialized @6445ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T19:04:57,673 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-18T19:04:57,673 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-18T19:04:57,674 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-18T19:04:57,674 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-18T19:04:57,675 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-18T19:04:57,676 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-18T19:04:57,677 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-18T19:04:57,677 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-18T19:04:57,679 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-18T19:04:57,738 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 36641 2024-11-18T19:04:57,739 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-18T19:04:57,752 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-18T19:04:57,754 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@bc9258d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:04:57,754 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-18T19:04:57,755 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@51ab1ff8{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-18T19:04:57,792 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-18T19:04:57,800 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:04:57,800 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:04:57,801 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T19:04:57,805 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-18T19:04:58,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@78825dd4{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-18T19:04:58,421 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7952237b{HTTP/1.1, (http/1.1)}{0.0.0.0:36641} 2024-11-18T19:04:58,421 INFO [Time-limited test {}] server.Server(415): Started @7239ms 2024-11-18T19:04:58,425 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9, hbase.cluster.distributed=false 2024-11-18T19:04:58,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-18T19:04:58,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35649 2024-11-18T19:04:58,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35649 2024-11-18T19:04:58,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35649 2024-11-18T19:04:58,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35649 2024-11-18T19:04:58,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35649 2024-11-18T19:04:58,580 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/818bff808382:0 server-side Connection retries=45 2024-11-18T19:04:58,582 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:04:58,582 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T19:04:58,583 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T19:04:58,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T19:04:58,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T19:04:58,587 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T19:04:58,589 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T19:04:58,590 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33643 2024-11-18T19:04:58,592 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33643 connecting to ZooKeeper ensemble=127.0.0.1:64381 2024-11-18T19:04:58,594 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:04:58,597 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:04:58,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:336430x0, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T19:04:58,603 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33643-0x1000c059bf10001 connected 2024-11-18T19:04:58,603 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-18T19:04:58,606 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-18T19:04:58,606 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-18T19:04:58,607 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-18T19:04:58,607 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-18T19:04:58,607 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-18T19:04:58,609 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-18T19:04:58,609 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-18T19:04:58,609 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-18T19:04:58,610 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-18T19:04:58,612 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 44621 2024-11-18T19:04:58,612 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-18T19:04:58,614 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-18T19:04:58,614 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1cdf8417{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:04:58,615 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-18T19:04:58,615 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@793157ae{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-18T19:04:58,628 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-18T19:04:58,629 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:04:58,630 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:04:58,630 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T19:04:58,631 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-18T19:04:58,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@2b61b876{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-18T19:04:58,635 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@25aa050e{HTTP/1.1, (http/1.1)}{0.0.0.0:44621} 2024-11-18T19:04:58,635 INFO [Time-limited test {}] server.Server(415): Started @7453ms 2024-11-18T19:04:58,640 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T19:04:58,648 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T19:04:58,650 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-18T19:04:58,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-18T19:04:58,657 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33643 2024-11-18T19:04:58,657 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33643 2024-11-18T19:04:58,658 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33643 2024-11-18T19:04:58,658 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33643 2024-11-18T19:04:58,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33643 2024-11-18T19:04:58,679 DEBUG [M:0;818bff808382:35649 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;818bff808382:35649 2024-11-18T19:04:58,682 INFO [master/818bff808382:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-18T19:04:58,688 INFO [master/818bff808382:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@4b1b14e2{HTTP/1.1, (http/1.1)}{0.0.0.0:37023} 2024-11-18T19:04:58,688 INFO [master/818bff808382:0:becomeActiveMaster {}] server.Server(415): Started @7506ms 2024-11-18T19:04:58,688 INFO [master/818bff808382:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/818bff808382,35649,1731956696665 2024-11-18T19:04:58,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-18T19:04:58,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-18T19:04:58,699 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/818bff808382,35649,1731956696665 2024-11-18T19:04:58,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-18T19:04:58,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:04:58,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:04:58,722 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-18T19:04:58,723 INFO [master/818bff808382:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/818bff808382,35649,1731956696665 from backup master directory 2024-11-18T19:04:58,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/818bff808382,35649,1731956696665 2024-11-18T19:04:58,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-18T19:04:58,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-18T19:04:58,727 WARN [master/818bff808382:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T19:04:58,727 INFO [master/818bff808382:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=818bff808382,35649,1731956696665 2024-11-18T19:04:58,729 INFO [master/818bff808382:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T19:04:58,731 INFO [master/818bff808382:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T19:04:58,789 DEBUG [master/818bff808382:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase.id] with ID: 6e087e80-4df5-4144-b1b9-0d02cb0c3509 2024-11-18T19:04:58,789 DEBUG [master/818bff808382:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.tmp/hbase.id 2024-11-18T19:04:58,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741826_1002 (size=42) 2024-11-18T19:04:59,202 DEBUG [master/818bff808382:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.tmp/hbase.id]:[hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase.id] 2024-11-18T19:04:59,248 INFO [master/818bff808382:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T19:04:59,253 INFO [master/818bff808382:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T19:04:59,274 INFO [master/818bff808382:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-18T19:04:59,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:04:59,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:04:59,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741827_1003 (size=196) 2024-11-18T19:04:59,712 INFO [master/818bff808382:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:04:59,715 INFO [master/818bff808382:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T19:04:59,722 INFO [master/818bff808382:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-18T19:04:59,724 INFO [master/818bff808382:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-18T19:04:59,736 DEBUG [master/818bff808382:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:04:59,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741828_1004 (size=1189) 2024-11-18T19:05:00,194 INFO [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store 2024-11-18T19:05:00,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741829_1005 (size=34) 2024-11-18T19:05:00,220 INFO [master/818bff808382:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T19:05:00,224 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:00,226 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T19:05:00,226 INFO [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:05:00,226 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:05:00,229 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T19:05:00,229 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:05:00,229 INFO [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:05:00,231 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731956700226Disabling compacts and flushes for region at 1731956700226Disabling writes for close at 1731956700229 (+3 ms)Writing region close event to WAL at 1731956700229Closed at 1731956700229 2024-11-18T19:05:00,233 WARN [master/818bff808382:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/.initializing 2024-11-18T19:05:00,233 DEBUG [master/818bff808382:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/WALs/818bff808382,35649,1731956696665 2024-11-18T19:05:00,238 INFO [master/818bff808382:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:05:00,243 INFO [master/818bff808382:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 818bff808382,35649,1731956696665.regiongroup-0 2024-11-18T19:05:00,261 INFO [master/818bff808382:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=818bff808382%2C35649%2C1731956696665.818bff808382%2C35649%2C1731956696665.regiongroup-0, suffix=, logDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/WALs/818bff808382,35649,1731956696665, archiveDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/oldWALs, maxLogs=10 2024-11-18T19:05:00,287 DEBUG [master/818bff808382:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/WALs/818bff808382,35649,1731956696665/818bff808382%2C35649%2C1731956696665.818bff808382%2C35649%2C1731956696665.regiongroup-0.1731956700266, exclude list is [], retry=0 2024-11-18T19:05:00,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:00,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T19:05:00,347 INFO [master/818bff808382:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/WALs/818bff808382,35649,1731956696665/818bff808382%2C35649%2C1731956696665.818bff808382%2C35649%2C1731956696665.regiongroup-0.1731956700266 2024-11-18T19:05:00,348 DEBUG [master/818bff808382:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:00,348 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:00,349 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:00,352 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,353 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,425 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T19:05:00,429 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:00,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:00,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T19:05:00,436 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:00,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:00,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T19:05:00,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:00,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:00,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T19:05:00,445 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:00,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:00,446 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,450 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,451 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,457 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,457 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,461 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T19:05:00,464 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T19:05:00,469 DEBUG [master/818bff808382:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:05:00,471 INFO [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10545524480, jitterRate=-0.017871499061584473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T19:05:00,477 DEBUG [master/818bff808382:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731956700370Initializing all the Stores at 1731956700372 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956700373 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956700374 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956700375 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956700375Cleaning up temporary data from old regions at 1731956700457 (+82 ms)Region opened successfully at 1731956700477 (+20 ms) 2024-11-18T19:05:00,478 INFO [master/818bff808382:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T19:05:00,509 DEBUG [master/818bff808382:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62081a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=818bff808382/172.17.0.2:0 2024-11-18T19:05:00,544 INFO [master/818bff808382:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T19:05:00,556 INFO [master/818bff808382:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T19:05:00,556 INFO [master/818bff808382:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T19:05:00,560 INFO [master/818bff808382:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T19:05:00,561 INFO [master/818bff808382:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T19:05:00,566 INFO [master/818bff808382:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T19:05:00,566 INFO [master/818bff808382:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T19:05:00,593 INFO [master/818bff808382:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T19:05:00,603 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-18T19:05:00,605 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-18T19:05:00,607 INFO [master/818bff808382:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T19:05:00,609 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-18T19:05:00,610 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-18T19:05:00,612 INFO [master/818bff808382:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T19:05:00,616 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-18T19:05:00,617 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-18T19:05:00,618 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-18T19:05:00,620 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-18T19:05:00,638 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T19:05:00,640 DEBUG [master/818bff808382:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-18T19:05:00,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-18T19:05:00,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-18T19:05:00,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:00,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:00,648 INFO [master/818bff808382:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=818bff808382,35649,1731956696665, sessionid=0x1000c059bf10000, setting cluster-up flag (Was=false) 2024-11-18T19:05:00,655 INFO [master/818bff808382:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-18T19:05:00,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:00,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:00,667 DEBUG [master/818bff808382:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-18T19:05:00,669 DEBUG [master/818bff808382:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=818bff808382,35649,1731956696665 2024-11-18T19:05:00,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:00,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:00,677 DEBUG [master/818bff808382:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-18T19:05:00,679 DEBUG [master/818bff808382:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=818bff808382,35649,1731956696665 2024-11-18T19:05:00,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:00,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:00,689 DEBUG [master/818bff808382:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-18T19:05:00,691 DEBUG [master/818bff808382:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=818bff808382,35649,1731956696665 2024-11-18T19:05:00,697 INFO [master/818bff808382:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T19:05:00,765 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(746): ClusterId : 6e087e80-4df5-4144-b1b9-0d02cb0c3509 2024-11-18T19:05:00,766 INFO [RS:0;818bff808382:33643 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-18T19:05:00,769 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T19:05:00,773 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T19:05:00,774 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-18T19:05:00,776 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-18T19:05:00,776 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T19:05:00,779 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T19:05:00,779 DEBUG [RS:0;818bff808382:33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bc4ca1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=818bff808382/172.17.0.2:0 2024-11-18T19:05:00,794 DEBUG [RS:0;818bff808382:33643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;818bff808382:33643 2024-11-18T19:05:00,797 INFO [RS:0;818bff808382:33643 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T19:05:00,797 INFO [RS:0;818bff808382:33643 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T19:05:00,797 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T19:05:00,799 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(2659): reportForDuty to master=818bff808382,35649,1731956696665 with port=33643, startcode=1731956698538 2024-11-18T19:05:00,801 DEBUG [master/818bff808382:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T19:05:00,810 DEBUG [RS:0;818bff808382:33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T19:05:00,810 INFO [master/818bff808382:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T19:05:00,817 INFO [master/818bff808382:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T19:05:00,822 DEBUG [master/818bff808382:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 818bff808382,35649,1731956696665 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T19:05:00,829 DEBUG [master/818bff808382:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/818bff808382:0, corePoolSize=5, maxPoolSize=5 2024-11-18T19:05:00,829 DEBUG [master/818bff808382:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/818bff808382:0, corePoolSize=5, maxPoolSize=5 2024-11-18T19:05:00,829 DEBUG [master/818bff808382:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/818bff808382:0, corePoolSize=5, maxPoolSize=5 2024-11-18T19:05:00,830 DEBUG [master/818bff808382:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/818bff808382:0, corePoolSize=5, maxPoolSize=5 2024-11-18T19:05:00,830 DEBUG [master/818bff808382:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/818bff808382:0, corePoolSize=10, maxPoolSize=10 2024-11-18T19:05:00,830 DEBUG [master/818bff808382:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:00,830 DEBUG [master/818bff808382:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/818bff808382:0, corePoolSize=2, maxPoolSize=2 2024-11-18T19:05:00,830 DEBUG [master/818bff808382:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:00,835 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T19:05:00,836 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T19:05:00,837 INFO [master/818bff808382:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731956730837 2024-11-18T19:05:00,838 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T19:05:00,839 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T19:05:00,841 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:00,841 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T19:05:00,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43583, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T19:05:00,845 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T19:05:00,845 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-18T19:05:00,846 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T19:05:00,846 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T19:05:00,846 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T19:05:00,847 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:00,850 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T19:05:00,852 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T19:05:00,852 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T19:05:00,852 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35649 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T19:05:00,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741831_1007 (size=1321) 2024-11-18T19:05:00,856 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T19:05:00,857 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:00,868 DEBUG [master/818bff808382:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fc123d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:00,868 DEBUG [master/818bff808382:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:00,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741832_1008 (size=32) 2024-11-18T19:05:00,872 DEBUG [master/818bff808382:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:00,879 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:00,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:00,884 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T19:05:00,884 WARN [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T19:05:00,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:00,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f91450e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:00,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:00,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:00,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:00,892 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:00,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2543c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:00,896 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-18T19:05:00,898 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T19:05:00,899 INFO [master/818bff808382:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T19:05:00,900 DEBUG [master/818bff808382:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/818bff808382:0:becomeActiveMaster-HFileCleaner.large.0-1731956700900,5,FailOnTimeoutGroup] 2024-11-18T19:05:00,901 DEBUG [master/818bff808382:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/818bff808382:0:becomeActiveMaster-HFileCleaner.small.0-1731956700901,5,FailOnTimeoutGroup] 2024-11-18T19:05:00,901 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:00,901 INFO [master/818bff808382:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T19:05:00,902 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:00,903 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:00,986 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(2659): reportForDuty to master=818bff808382,35649,1731956696665 with port=33643, startcode=1731956698538 2024-11-18T19:05:00,989 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35649 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 818bff808382,33643,1731956698538 2024-11-18T19:05:00,992 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35649 {}] master.ServerManager(517): Registering regionserver=818bff808382,33643,1731956698538 2024-11-18T19:05:01,003 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:01,003 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45363 2024-11-18T19:05:01,003 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=36641 2024-11-18T19:05:01,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-18T19:05:01,011 DEBUG [RS:0;818bff808382:33643 {}] zookeeper.ZKUtil(111): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rs/818bff808382,33643,1731956698538 2024-11-18T19:05:01,011 WARN [RS:0;818bff808382:33643 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T19:05:01,012 INFO [RS:0;818bff808382:33643 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-18T19:05:01,012 INFO [RS:0;818bff808382:33643 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-18T19:05:01,012 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538 2024-11-18T19:05:01,014 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [818bff808382,33643,1731956698538] 2024-11-18T19:05:01,038 INFO [RS:0;818bff808382:33643 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T19:05:01,051 INFO [RS:0;818bff808382:33643 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T19:05:01,056 INFO [RS:0;818bff808382:33643 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T19:05:01,056 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,057 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T19:05:01,063 INFO [RS:0;818bff808382:33643 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T19:05:01,064 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,064 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,064 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,064 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/818bff808382:0, corePoolSize=2, maxPoolSize=2 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,065 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/818bff808382:0, corePoolSize=1, maxPoolSize=1 2024-11-18T19:05:01,066 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0, corePoolSize=3, maxPoolSize=3 2024-11-18T19:05:01,066 DEBUG [RS:0;818bff808382:33643 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/818bff808382:0, corePoolSize=3, maxPoolSize=3 2024-11-18T19:05:01,067 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,067 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,067 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,067 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,067 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,067 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=818bff808382,33643,1731956698538-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T19:05:01,086 INFO [RS:0;818bff808382:33643 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T19:05:01,088 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=818bff808382,33643,1731956698538-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,089 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,089 INFO [RS:0;818bff808382:33643 {}] regionserver.Replication(171): 818bff808382,33643,1731956698538 started 2024-11-18T19:05:01,107 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,107 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1482): Serving as 818bff808382,33643,1731956698538, RpcServer on 818bff808382/172.17.0.2:33643, sessionid=0x1000c059bf10001 2024-11-18T19:05:01,108 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T19:05:01,108 DEBUG [RS:0;818bff808382:33643 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 818bff808382,33643,1731956698538 2024-11-18T19:05:01,109 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '818bff808382,33643,1731956698538' 2024-11-18T19:05:01,109 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-18T19:05:01,110 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-18T19:05:01,111 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T19:05:01,111 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-18T19:05:01,111 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '818bff808382,33643,1731956698538' 2024-11-18T19:05:01,111 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:05:01,111 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:05:01,112 INFO [RS:0;818bff808382:33643 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-18T19:05:01,112 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-18T19:05:01,112 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T19:05:01,112 DEBUG [RS:0;818bff808382:33643 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 818bff808382,33643,1731956698538 2024-11-18T19:05:01,112 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '818bff808382,33643,1731956698538' 2024-11-18T19:05:01,112 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-18T19:05:01,113 DEBUG [RS:0;818bff808382:33643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-18T19:05:01,114 DEBUG [RS:0;818bff808382:33643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T19:05:01,114 INFO [RS:0;818bff808382:33643 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T19:05:01,114 INFO [RS:0;818bff808382:33643 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T19:05:01,220 INFO [RS:0;818bff808382:33643 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:05:01,221 INFO [RS:0;818bff808382:33643 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 818bff808382,33643,1731956698538.regiongroup-0 2024-11-18T19:05:01,224 INFO [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0, suffix=, logDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538, archiveDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs, maxLogs=32 2024-11-18T19:05:01,240 DEBUG [RS:0;818bff808382:33643 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956701226, exclude list is [], retry=0 2024-11-18T19:05:01,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:01,248 INFO [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956701226 2024-11-18T19:05:01,248 DEBUG [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:01,273 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:01,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T19:05:01,278 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T19:05:01,278 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:01,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:01,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T19:05:01,281 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T19:05:01,281 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:01,282 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:01,282 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T19:05:01,284 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T19:05:01,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:01,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:01,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T19:05:01,287 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T19:05:01,287 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:01,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:01,288 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T19:05:01,289 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740 2024-11-18T19:05:01,290 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740 2024-11-18T19:05:01,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T19:05:01,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T19:05:01,293 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T19:05:01,295 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T19:05:01,299 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:05:01,299 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10496802240, jitterRate=-0.022409111261367798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T19:05:01,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731956701274Initializing all the Stores at 1731956701275 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956701275Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956701276 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956701276Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956701276Cleaning up temporary data from old regions at 1731956701293 (+17 ms)Region opened successfully at 1731956701302 (+9 ms) 2024-11-18T19:05:01,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T19:05:01,302 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T19:05:01,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T19:05:01,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T19:05:01,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T19:05:01,303 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T19:05:01,303 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731956701302Disabling compacts and flushes for region at 1731956701302Disabling writes for close at 1731956701302Writing region close event to WAL at 1731956701303 (+1 ms)Closed at 1731956701303 2024-11-18T19:05:01,307 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T19:05:01,307 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T19:05:01,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T19:05:01,322 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:05:01,324 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T19:05:01,477 DEBUG [818bff808382:35649 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T19:05:01,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:01,498 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 818bff808382,33643,1731956698538, state=OPENING 2024-11-18T19:05:01,503 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T19:05:01,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:01,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:05:01,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-18T19:05:01,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-18T19:05:01,507 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T19:05:01,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:01,684 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T19:05:01,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43675, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T19:05:01,695 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T19:05:01,696 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-18T19:05:01,696 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-18T19:05:01,696 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:05:01,697 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T19:05:01,700 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=818bff808382%2C33643%2C1731956698538.meta, suffix=.meta, logDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538, archiveDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs, maxLogs=32 2024-11-18T19:05:01,715 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956701701.meta, exclude list is [], retry=0 2024-11-18T19:05:01,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:01,721 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956701701.meta 2024-11-18T19:05:01,722 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:01,722 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:01,726 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:01,727 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T19:05:01,730 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T19:05:01,730 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T19:05:01,734 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T19:05:01,734 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:01,734 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T19:05:01,734 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T19:05:01,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T19:05:01,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T19:05:01,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:01,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:01,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T19:05:01,741 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T19:05:01,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:01,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:01,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T19:05:01,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T19:05:01,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:01,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:01,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T19:05:01,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T19:05:01,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:01,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T19:05:01,746 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T19:05:01,747 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740 2024-11-18T19:05:01,750 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740 2024-11-18T19:05:01,752 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T19:05:01,752 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T19:05:01,753 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T19:05:01,755 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T19:05:01,757 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10664524160, jitterRate=-0.006788790225982666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T19:05:01,757 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T19:05:01,758 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731956701735Writing region info on filesystem at 1731956701735Initializing all the Stores at 1731956701737 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956701737Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956701737Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956701737Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956701737Cleaning up temporary data from old regions at 1731956701752 (+15 ms)Running coprocessor post-open hooks at 1731956701757 (+5 ms)Region opened successfully at 1731956701758 (+1 ms) 2024-11-18T19:05:01,765 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731956701675 2024-11-18T19:05:01,777 DEBUG [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T19:05:01,777 INFO [RS_OPEN_META-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T19:05:01,779 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:01,781 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 818bff808382,33643,1731956698538, state=OPEN 2024-11-18T19:05:01,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-18T19:05:01,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-18T19:05:01,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-18T19:05:01,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-18T19:05:01,785 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=818bff808382,33643,1731956698538 2024-11-18T19:05:01,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T19:05:01,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=818bff808382,33643,1731956698538 in 276 msec 2024-11-18T19:05:01,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T19:05:01,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 479 msec 2024-11-18T19:05:01,801 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T19:05:01,801 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T19:05:01,828 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:01,829 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:01,850 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:01,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55465, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:01,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1250 sec 2024-11-18T19:05:01,873 INFO [master/818bff808382:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731956701873, completionTime=-1 2024-11-18T19:05:01,875 INFO [master/818bff808382:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T19:05:01,875 DEBUG [master/818bff808382:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-18T19:05:01,905 INFO [master/818bff808382:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-18T19:05:01,905 INFO [master/818bff808382:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731956761905 2024-11-18T19:05:01,905 INFO [master/818bff808382:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731956821905 2024-11-18T19:05:01,905 INFO [master/818bff808382:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 29 msec 2024-11-18T19:05:01,907 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=818bff808382,35649,1731956696665-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,907 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=818bff808382,35649,1731956696665-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,908 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=818bff808382,35649,1731956696665-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,909 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-818bff808382:35649, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,910 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,910 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,920 DEBUG [master/818bff808382:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T19:05:01,945 INFO [master/818bff808382:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.218sec 2024-11-18T19:05:01,947 INFO [master/818bff808382:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T19:05:01,949 INFO [master/818bff808382:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T19:05:01,950 INFO [master/818bff808382:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T19:05:01,951 INFO [master/818bff808382:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T19:05:01,951 INFO [master/818bff808382:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T19:05:01,952 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=818bff808382,35649,1731956696665-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T19:05:01,952 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=818bff808382,35649,1731956696665-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T19:05:01,961 DEBUG [master/818bff808382:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T19:05:01,962 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T19:05:01,962 INFO [master/818bff808382:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=818bff808382,35649,1731956696665-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T19:05:01,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c0ed397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:01,968 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T19:05:01,968 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T19:05:01,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:01,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:01,981 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d201cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:01,986 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:01,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4106f58a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:01,988 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:02,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:02,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=818bff808382,35649,1731956696665 2024-11-18T19:05:02,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/test.cache.data in system properties and HBase conf 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir in system properties and HBase conf 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T19:05:02,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/nfs.dump.dir in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/java.io.tmpdir in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T19:05:02,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T19:05:02,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T19:05:02,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741835_1011 (size=592039) 2024-11-18T19:05:02,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741836_1012 (size=1663647) 2024-11-18T19:05:03,631 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:05:03,729 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:05:03,934 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:05:03,938 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-18T19:05:03,938 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:05:03,940 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:05:03,940 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:05:03,940 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T19:05:03,941 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45067f3b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:05:03,941 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a80b569{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-18T19:05:03,942 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:05:03,942 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:05:03,942 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T19:05:03,944 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:05:03,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c57c6a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:05:03,955 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ca0b593{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-18T19:05:04,086 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2024-11-18T19:05:04,086 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2024-11-18T19:05:04,086 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-18T19:05:04,088 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-18T19:05:04,126 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:04,216 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:04,417 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:04,436 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5dc6a938{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/java.io.tmpdir/jetty-localhost-33303-hadoop-yarn-common-3_4_1_jar-_-any-8869318435238157178/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-18T19:05:04,436 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@355625e7{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/java.io.tmpdir/jetty-localhost-43737-hadoop-yarn-common-3_4_1_jar-_-any-9353235146098066916/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-18T19:05:04,437 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@73ba90c4{HTTP/1.1, (http/1.1)}{localhost:43737} 2024-11-18T19:05:04,437 INFO [Thread-251 {}] server.Server(415): Started @13255ms 2024-11-18T19:05:04,437 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@37a72b0b{HTTP/1.1, (http/1.1)}{localhost:33303} 2024-11-18T19:05:04,437 INFO [Time-limited test {}] server.Server(415): Started @13255ms 2024-11-18T19:05:04,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741837_1013 (size=5) 2024-11-18T19:05:05,497 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-18T19:05:05,502 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:05:05,534 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-18T19:05:05,535 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:05:05,540 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:05:05,540 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:05:05,540 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T19:05:05,541 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:05:05,542 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3cf29358{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:05:05,542 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25e99e8f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-18T19:05:05,601 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-18T19:05:05,601 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-18T19:05:05,601 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-18T19:05:05,601 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-18T19:05:05,610 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:05,616 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:05,716 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:05,726 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ceda6b8{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/java.io.tmpdir/jetty-localhost-34267-hadoop-yarn-common-3_4_1_jar-_-any-12481306104278668572/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-18T19:05:05,727 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@b86af18{HTTP/1.1, (http/1.1)}{localhost:34267} 2024-11-18T19:05:05,727 INFO [Time-limited test {}] server.Server(415): Started @14545ms 2024-11-18T19:05:05,862 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-18T19:05:05,865 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:05:05,877 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-18T19:05:05,878 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T19:05:05,881 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T19:05:05,882 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T19:05:05,882 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T19:05:05,882 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T19:05:05,883 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@297e0e95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,AVAILABLE} 2024-11-18T19:05:05,883 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@729dadbb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-18T19:05:05,936 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-18T19:05:05,937 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-18T19:05:05,937 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-18T19:05:05,937 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-18T19:05:05,945 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:05,951 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:06,070 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-18T19:05:06,075 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7843287{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/java.io.tmpdir/jetty-localhost-36495-hadoop-yarn-common-3_4_1_jar-_-any-12098953219050384842/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-18T19:05:06,076 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@67510acc{HTTP/1.1, (http/1.1)}{localhost:36495} 2024-11-18T19:05:06,077 INFO [Time-limited test {}] server.Server(415): Started @14894ms 2024-11-18T19:05:06,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-18T19:05:06,111 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:45363/backupUT 2024-11-18T19:05:06,115 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:05:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:05:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33243812 2024-11-18T19:05:06,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:05:06,127 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:05:06,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-11-18T19:05:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-18T19:05:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T19:05:06,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 30 msec 2024-11-18T19:05:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T19:05:06,264 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-18T19:05:06,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-11-18T19:05:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-18T19:05:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-18T19:05:06,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2024-11-18T19:05:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-18T19:05:06,382 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-18T19:05:06,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-11-18T19:05:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-18T19:05:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-18T19:05:06,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 14 msec 2024-11-18T19:05:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-18T19:05:06,493 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-18T19:05:06,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-11-18T19:05:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-18T19:05:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T19:05:06,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 12 msec 2024-11-18T19:05:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T19:05:06,603 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-18T19:05:06,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1731956706111', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1731956706111 2024-11-18T19:05:06,623 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1731956706111 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:05:06,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1731956706111" procId is: 8 2024-11-18T19:05:06,625 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-18T19:05:06,627 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1731956706111 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:05:06,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741838_1014 (size=367) 2024-11-18T19:05:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-18T19:05:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-18T19:05:07,042 INFO [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b38dbac143e9407aa48a95e23085ace3, NAME => 'test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1731956706111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:07,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741839_1015 (size=53) 2024-11-18T19:05:07,247 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-18T19:05:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-18T19:05:07,301 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T19:05:07,303 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T19:05:07,303 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T19:05:07,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:05:07,305 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-18T19:05:07,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T19:05:07,305 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T19:05:07,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T19:05:07,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T19:05:07,453 DEBUG [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(898): Instantiated test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:07,454 DEBUG [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(1722): Closing b38dbac143e9407aa48a95e23085ace3, disabling compactions & flushes 2024-11-18T19:05:07,454 INFO [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(1755): Closing region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:07,454 DEBUG [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:07,454 DEBUG [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. after waiting 0 ms 2024-11-18T19:05:07,454 DEBUG [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:07,454 INFO [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(1973): Closed test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:07,454 DEBUG [RegionOpenAndInit-test-1731956706111-pool-0 {}] regionserver.HRegion(1676): Region close journal for b38dbac143e9407aa48a95e23085ace3: Waiting for close lock at 1731956707454Disabling compacts and flushes for region at 1731956707454Disabling writes for close at 1731956707454Writing region close event to WAL at 1731956707454Closed at 1731956707454 2024-11-18T19:05:07,456 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1731956706111 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:05:07,462 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731956707456"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956707456"}]},"ts":"1731956707456"} 2024-11-18T19:05:07,467 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T19:05:07,469 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1731956706111 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:05:07,472 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1731956706111","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956707469"}]},"ts":"1731956707469"} 2024-11-18T19:05:07,476 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1731956706111, state=ENABLING in hbase:meta 2024-11-18T19:05:07,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, ASSIGN}] 2024-11-18T19:05:07,480 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, ASSIGN 2024-11-18T19:05:07,481 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:05:07,633 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:07,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, ASSIGN because future has completed 2024-11-18T19:05:07,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-18T19:05:07,815 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:07,815 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T19:05:07,816 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor 818bff808382,33643,1731956698538.regiongroup-1 2024-11-18T19:05:07,819 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1, suffix=, logDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538, archiveDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs, maxLogs=32 2024-11-18T19:05:07,835 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956707820, exclude list is [], retry=0 2024-11-18T19:05:07,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:07,841 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956707820 2024-11-18T19:05:07,842 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:07,842 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => b38dbac143e9407aa48a95e23085ace3, NAME => 'test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:07,842 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:07,842 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731956706111 b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,843 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:07,843 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,843 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,845 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,847 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b38dbac143e9407aa48a95e23085ace3 columnFamilyName f 2024-11-18T19:05:07,847 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:07,848 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(327): Store=b38dbac143e9407aa48a95e23085ace3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:07,848 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,850 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,851 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,851 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,851 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,854 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,858 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:05:07,858 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened b38dbac143e9407aa48a95e23085ace3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9595326880, jitterRate=-0.10636554658412933}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:05:07,858 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:07,860 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for b38dbac143e9407aa48a95e23085ace3: Running coprocessor pre-open hook at 1731956707843Writing region info on filesystem at 1731956707843Initializing all the Stores at 1731956707845 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956707845Cleaning up temporary data from old regions at 1731956707851 (+6 ms)Running coprocessor post-open hooks at 1731956707858 (+7 ms)Region opened successfully at 1731956707859 (+1 ms) 2024-11-18T19:05:07,861 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., pid=10, masterSystemTime=1731956707791 2024-11-18T19:05:07,865 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:07,865 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:07,866 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:07,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:07,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-18T19:05:07,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 in 233 msec 2024-11-18T19:05:07,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-18T19:05:07,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, ASSIGN in 397 msec 2024-11-18T19:05:07,880 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1731956706111 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:05:07,880 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1731956706111","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956707880"}]},"ts":"1731956707880"} 2024-11-18T19:05:07,883 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1731956706111, state=ENABLED in hbase:meta 2024-11-18T19:05:07,884 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1731956706111 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:05:07,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1731956706111 in 1.2680 sec 2024-11-18T19:05:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-18T19:05:08,773 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1731956706111 completed 2024-11-18T19:05:08,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79736906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:08,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:08,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:08,776 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4185f164, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:08,779 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@115a5a14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:08,783 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:08,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:08,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731956706111', row='row0', locateType=CURRENT is [region=test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:05:08,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegion(8528): writing data to region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. with WAL disabled. Data may be lost in the event of a crash. 2024-11-18T19:05:08,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17319567061111', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17319567061111 2024-11-18T19:05:08,943 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17319567061111 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:05:08,943 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:08,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17319567061111" procId is: 11 2024-11-18T19:05:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T19:05:08,945 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17319567061111 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:05:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741841_1017 (size=364) 2024-11-18T19:05:08,956 INFO [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d144bc0c998ecf0ed9ce62cb631ee3eb, NAME => 'ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17319567061111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:08,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741842_1018 (size=50) 2024-11-18T19:05:08,965 DEBUG [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:08,965 DEBUG [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(1722): Closing d144bc0c998ecf0ed9ce62cb631ee3eb, disabling compactions & flushes 2024-11-18T19:05:08,965 INFO [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:08,965 DEBUG [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:08,965 DEBUG [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. after waiting 0 ms 2024-11-18T19:05:08,965 DEBUG [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:08,965 INFO [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:08,966 DEBUG [RegionOpenAndInit-ns2:test-17319567061111-pool-0 {}] regionserver.HRegion(1676): Region close journal for d144bc0c998ecf0ed9ce62cb631ee3eb: Waiting for close lock at 1731956708965Disabling compacts and flushes for region at 1731956708965Disabling writes for close at 1731956708965Writing region close event to WAL at 1731956708965Closed at 1731956708965 2024-11-18T19:05:08,967 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17319567061111 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:05:08,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731956708968"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956708968"}]},"ts":"1731956708968"} 2024-11-18T19:05:08,971 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T19:05:08,973 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17319567061111 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:05:08,973 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17319567061111","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956708973"}]},"ts":"1731956708973"} 2024-11-18T19:05:08,976 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17319567061111, state=ENABLING in hbase:meta 2024-11-18T19:05:08,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17319567061111, region=d144bc0c998ecf0ed9ce62cb631ee3eb, ASSIGN}] 2024-11-18T19:05:08,978 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17319567061111, region=d144bc0c998ecf0ed9ce62cb631ee3eb, ASSIGN 2024-11-18T19:05:08,979 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17319567061111, region=d144bc0c998ecf0ed9ce62cb631ee3eb, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:05:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T19:05:09,130 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=d144bc0c998ecf0ed9ce62cb631ee3eb, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:09,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17319567061111, region=d144bc0c998ecf0ed9ce62cb631ee3eb, ASSIGN because future has completed 2024-11-18T19:05:09,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T19:05:09,295 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:09,296 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => d144bc0c998ecf0ed9ce62cb631ee3eb, NAME => 'ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:09,296 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:09,296 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17319567061111 d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,296 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:09,297 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,297 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,299 INFO [StoreOpener-d144bc0c998ecf0ed9ce62cb631ee3eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,301 INFO [StoreOpener-d144bc0c998ecf0ed9ce62cb631ee3eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d144bc0c998ecf0ed9ce62cb631ee3eb columnFamilyName f 2024-11-18T19:05:09,301 DEBUG [StoreOpener-d144bc0c998ecf0ed9ce62cb631ee3eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:09,302 INFO [StoreOpener-d144bc0c998ecf0ed9ce62cb631ee3eb-1 {}] regionserver.HStore(327): Store=d144bc0c998ecf0ed9ce62cb631ee3eb/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:09,302 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,303 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,304 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,305 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,305 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,308 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,311 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:05:09,312 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened d144bc0c998ecf0ed9ce62cb631ee3eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10305380160, jitterRate=-0.04023668169975281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:05:09,312 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:09,313 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for d144bc0c998ecf0ed9ce62cb631ee3eb: Running coprocessor pre-open hook at 1731956709297Writing region info on filesystem at 1731956709297Initializing all the Stores at 1731956709299 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956709299Cleaning up temporary data from old regions at 1731956709305 (+6 ms)Running coprocessor post-open hooks at 1731956709312 (+7 ms)Region opened successfully at 1731956709313 (+1 ms) 2024-11-18T19:05:09,315 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb., pid=13, masterSystemTime=1731956709289 2024-11-18T19:05:09,318 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:09,318 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:09,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=d144bc0c998ecf0ed9ce62cb631ee3eb, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:09,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:09,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-18T19:05:09,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb, server=818bff808382,33643,1731956698538 in 189 msec 2024-11-18T19:05:09,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T19:05:09,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17319567061111, region=d144bc0c998ecf0ed9ce62cb631ee3eb, ASSIGN in 352 msec 2024-11-18T19:05:09,333 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17319567061111 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:05:09,334 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17319567061111","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956709333"}]},"ts":"1731956709333"} 2024-11-18T19:05:09,336 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17319567061111, state=ENABLED in hbase:meta 2024-11-18T19:05:09,337 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17319567061111 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:05:09,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17319567061111 in 397 msec 2024-11-18T19:05:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T19:05:09,573 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17319567061111 completed 2024-11-18T19:05:09,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17319567061111', row='row0', locateType=CURRENT is [region=ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:05:09,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegion(8528): writing data to region ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. with WAL disabled. Data may be lost in the event of a crash. 2024-11-18T19:05:09,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17319567061112', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17319567061112 2024-11-18T19:05:09,718 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17319567061112 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:05:09,718 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:09,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17319567061112" procId is: 14 2024-11-18T19:05:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-18T19:05:09,720 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17319567061112 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:05:09,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741843_1019 (size=390) 2024-11-18T19:05:09,731 INFO [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f44dca6ed6ddff070a4eead3e92d66bc, NAME => 'ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17319567061112', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:09,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741844_1020 (size=50) 2024-11-18T19:05:09,739 DEBUG [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:09,739 DEBUG [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(1722): Closing f44dca6ed6ddff070a4eead3e92d66bc, disabling compactions & flushes 2024-11-18T19:05:09,739 INFO [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:05:09,739 DEBUG [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:05:09,739 DEBUG [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. after waiting 0 ms 2024-11-18T19:05:09,739 DEBUG [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:05:09,739 INFO [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:05:09,739 DEBUG [RegionOpenAndInit-ns3:test-17319567061112-pool-0 {}] regionserver.HRegion(1676): Region close journal for f44dca6ed6ddff070a4eead3e92d66bc: Waiting for close lock at 1731956709739Disabling compacts and flushes for region at 1731956709739Disabling writes for close at 1731956709739Writing region close event to WAL at 1731956709739Closed at 1731956709739 2024-11-18T19:05:09,740 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17319567061112 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:05:09,741 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731956709741"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956709741"}]},"ts":"1731956709741"} 2024-11-18T19:05:09,744 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T19:05:09,745 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17319567061112 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:05:09,745 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17319567061112","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956709745"}]},"ts":"1731956709745"} 2024-11-18T19:05:09,747 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17319567061112, state=ENABLING in hbase:meta 2024-11-18T19:05:09,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17319567061112, region=f44dca6ed6ddff070a4eead3e92d66bc, ASSIGN}] 2024-11-18T19:05:09,750 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17319567061112, region=f44dca6ed6ddff070a4eead3e92d66bc, ASSIGN 2024-11-18T19:05:09,751 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17319567061112, region=f44dca6ed6ddff070a4eead3e92d66bc, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:05:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-18T19:05:09,902 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f44dca6ed6ddff070a4eead3e92d66bc, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:09,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17319567061112, region=f44dca6ed6ddff070a4eead3e92d66bc, ASSIGN because future has completed 2024-11-18T19:05:09,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f44dca6ed6ddff070a4eead3e92d66bc, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-18T19:05:10,063 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:05:10,064 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => f44dca6ed6ddff070a4eead3e92d66bc, NAME => 'ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:10,064 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:10,064 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17319567061112 f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,064 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:10,065 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,065 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,067 INFO [StoreOpener-f44dca6ed6ddff070a4eead3e92d66bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,069 INFO [StoreOpener-f44dca6ed6ddff070a4eead3e92d66bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f44dca6ed6ddff070a4eead3e92d66bc columnFamilyName f 2024-11-18T19:05:10,069 DEBUG [StoreOpener-f44dca6ed6ddff070a4eead3e92d66bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:10,070 INFO [StoreOpener-f44dca6ed6ddff070a4eead3e92d66bc-1 {}] regionserver.HStore(327): Store=f44dca6ed6ddff070a4eead3e92d66bc/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:10,070 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,071 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns3/test-17319567061112/f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,071 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns3/test-17319567061112/f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,072 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,072 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,075 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,078 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns3/test-17319567061112/f44dca6ed6ddff070a4eead3e92d66bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:05:10,078 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened f44dca6ed6ddff070a4eead3e92d66bc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10676499840, jitterRate=-0.005673468112945557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:05:10,079 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:05:10,080 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for f44dca6ed6ddff070a4eead3e92d66bc: Running coprocessor pre-open hook at 1731956710065Writing region info on filesystem at 1731956710065Initializing all the Stores at 1731956710066 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956710066Cleaning up temporary data from old regions at 1731956710072 (+6 ms)Running coprocessor post-open hooks at 1731956710079 (+7 ms)Region opened successfully at 1731956710080 (+1 ms) 2024-11-18T19:05:10,081 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc., pid=16, masterSystemTime=1731956710058 2024-11-18T19:05:10,084 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:05:10,085 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:05:10,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f44dca6ed6ddff070a4eead3e92d66bc, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:10,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f44dca6ed6ddff070a4eead3e92d66bc, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:10,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-18T19:05:10,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure f44dca6ed6ddff070a4eead3e92d66bc, server=818bff808382,33643,1731956698538 in 189 msec 2024-11-18T19:05:10,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-18T19:05:10,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17319567061112, region=f44dca6ed6ddff070a4eead3e92d66bc, ASSIGN in 351 msec 2024-11-18T19:05:10,104 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17319567061112 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:05:10,105 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17319567061112","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956710104"}]},"ts":"1731956710104"} 2024-11-18T19:05:10,107 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17319567061112, state=ENABLED in hbase:meta 2024-11-18T19:05:10,109 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17319567061112 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:05:10,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17319567061112 in 395 msec 2024-11-18T19:05:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-18T19:05:10,343 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17319567061112 completed 2024-11-18T19:05:10,343 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17319567061112 get assigned. Timeout = 60000ms 2024-11-18T19:05:10,345 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:05:10,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17319567061112 assigned to meta. Checking AM states. 2024-11-18T19:05:10,351 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:05:10,351 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17319567061112 assigned. 2024-11-18T19:05:10,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17319567061113', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17319567061113 2024-11-18T19:05:10,357 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17319567061113 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:05:10,357 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:10,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17319567061113" procId is: 17 2024-11-18T19:05:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T19:05:10,358 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17319567061113 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:05:10,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741845_1021 (size=390) 2024-11-18T19:05:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T19:05:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T19:05:10,770 INFO [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f88044664c926e5a290147e827b7cf95, NAME => 'ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17319567061113', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:10,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741846_1022 (size=50) 2024-11-18T19:05:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T19:05:11,179 DEBUG [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:11,179 DEBUG [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(1722): Closing f88044664c926e5a290147e827b7cf95, disabling compactions & flushes 2024-11-18T19:05:11,180 INFO [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:05:11,180 DEBUG [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:05:11,180 DEBUG [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. after waiting 0 ms 2024-11-18T19:05:11,180 DEBUG [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:05:11,180 INFO [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:05:11,180 DEBUG [RegionOpenAndInit-ns4:test-17319567061113-pool-0 {}] regionserver.HRegion(1676): Region close journal for f88044664c926e5a290147e827b7cf95: Waiting for close lock at 1731956711179Disabling compacts and flushes for region at 1731956711179Disabling writes for close at 1731956711180 (+1 ms)Writing region close event to WAL at 1731956711180Closed at 1731956711180 2024-11-18T19:05:11,182 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17319567061113 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:05:11,182 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731956711182"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956711182"}]},"ts":"1731956711182"} 2024-11-18T19:05:11,185 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T19:05:11,186 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17319567061113 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:05:11,187 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17319567061113","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956711186"}]},"ts":"1731956711186"} 2024-11-18T19:05:11,189 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17319567061113, state=ENABLING in hbase:meta 2024-11-18T19:05:11,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17319567061113, region=f88044664c926e5a290147e827b7cf95, ASSIGN}] 2024-11-18T19:05:11,191 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17319567061113, region=f88044664c926e5a290147e827b7cf95, ASSIGN 2024-11-18T19:05:11,192 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17319567061113, region=f88044664c926e5a290147e827b7cf95, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:05:11,343 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f88044664c926e5a290147e827b7cf95, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:11,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17319567061113, region=f88044664c926e5a290147e827b7cf95, ASSIGN because future has completed 2024-11-18T19:05:11,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f88044664c926e5a290147e827b7cf95, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T19:05:11,506 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:05:11,506 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => f88044664c926e5a290147e827b7cf95, NAME => 'ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:11,506 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:11,507 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17319567061113 f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,507 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:11,507 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,507 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,509 INFO [StoreOpener-f88044664c926e5a290147e827b7cf95-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,512 INFO [StoreOpener-f88044664c926e5a290147e827b7cf95-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f88044664c926e5a290147e827b7cf95 columnFamilyName f 2024-11-18T19:05:11,513 DEBUG [StoreOpener-f88044664c926e5a290147e827b7cf95-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:11,513 INFO [StoreOpener-f88044664c926e5a290147e827b7cf95-1 {}] regionserver.HStore(327): Store=f88044664c926e5a290147e827b7cf95/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:11,514 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,515 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns4/test-17319567061113/f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,515 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns4/test-17319567061113/f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,516 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,516 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,518 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,521 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns4/test-17319567061113/f88044664c926e5a290147e827b7cf95/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:05:11,522 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened f88044664c926e5a290147e827b7cf95; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10131956320, jitterRate=-0.05638803541660309}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:05:11,522 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f88044664c926e5a290147e827b7cf95 2024-11-18T19:05:11,523 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for f88044664c926e5a290147e827b7cf95: Running coprocessor pre-open hook at 1731956711507Writing region info on filesystem at 1731956711507Initializing all the Stores at 1731956711509 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956711509Cleaning up temporary data from old regions at 1731956711516 (+7 ms)Running coprocessor post-open hooks at 1731956711522 (+6 ms)Region opened successfully at 1731956711523 (+1 ms) 2024-11-18T19:05:11,525 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95., pid=19, masterSystemTime=1731956711500 2024-11-18T19:05:11,527 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:05:11,527 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:05:11,529 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f88044664c926e5a290147e827b7cf95, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:11,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f88044664c926e5a290147e827b7cf95, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:11,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-18T19:05:11,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure f88044664c926e5a290147e827b7cf95, server=818bff808382,33643,1731956698538 in 186 msec 2024-11-18T19:05:11,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T19:05:11,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17319567061113, region=f88044664c926e5a290147e827b7cf95, ASSIGN in 347 msec 2024-11-18T19:05:11,540 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17319567061113 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:05:11,540 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17319567061113","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956711540"}]},"ts":"1731956711540"} 2024-11-18T19:05:11,542 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17319567061113, state=ENABLED in hbase:meta 2024-11-18T19:05:11,544 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17319567061113 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:05:11,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17319567061113 in 1.1900 sec 2024-11-18T19:05:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T19:05:12,513 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17319567061113 completed 2024-11-18T19:05:12,513 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17319567061113 get assigned. Timeout = 60000ms 2024-11-18T19:05:12,514 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:05:12,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17319567061113 assigned to meta. Checking AM states. 2024-11-18T19:05:12,518 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:05:12,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17319567061113 assigned. 2024-11-18T19:05:12,519 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:05:12,519 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:05:12,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:12,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:12,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:12,595 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=9, AvailableMemoryMB=10962 2024-11-18T19:05:12,598 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-11-18T19:05:12,599 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-18T19:05:12,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1731956706111 from 'test-1731956706111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1731956706111', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1731956706111 2024-11-18T19:05:12,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741847_1023 (size=908) 2024-11-18T19:05:13,045 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/.tabledesc/.tableinfo.0000000001.367 2024-11-18T19:05:13,045 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/.tabledesc/.tableinfo.0000000002.908 2024-11-18T19:05:13,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1731956706111}] 2024-11-18T19:05:13,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE}] 2024-11-18T19:05:13,061 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE 2024-11-18T19:05:13,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=CLOSING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:13,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE because future has completed 2024-11-18T19:05:13,065 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T19:05:13,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:13,223 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:13,223 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T19:05:13,224 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing b38dbac143e9407aa48a95e23085ace3, disabling compactions & flushes 2024-11-18T19:05:13,224 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:13,224 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:13,224 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. after waiting 0 ms 2024-11-18T19:05:13,224 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:13,228 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing b38dbac143e9407aa48a95e23085ace3 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-18T19:05:13,299 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f/b697bbeabff04e5e8e1b821b7421fa7d is 37, key is row10/f:q1/1731956708812/Put/seqid=0 2024-11-18T19:05:13,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741848_1024 (size=8413) 2024-11-18T19:05:13,346 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-18T19:05:13,378 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1731956706111' 2024-11-18T19:05:13,378 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17319567061111' 2024-11-18T19:05:13,379 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17319567061113' 2024-11-18T19:05:13,379 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17319567061112' 2024-11-18T19:05:13,713 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f/b697bbeabff04e5e8e1b821b7421fa7d 2024-11-18T19:05:13,776 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f/b697bbeabff04e5e8e1b821b7421fa7d as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d 2024-11-18T19:05:13,786 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d, entries=99, sequenceid=103, filesize=8.2 K 2024-11-18T19:05:13,793 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for b38dbac143e9407aa48a95e23085ace3 in 564ms, sequenceid=103, compaction requested=false 2024-11-18T19:05:13,800 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-18T19:05:13,804 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:05:13,804 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:13,804 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for b38dbac143e9407aa48a95e23085ace3: Waiting for close lock at 1731956713224Running coprocessor pre-close hooks at 1731956713224Disabling compacts and flushes for region at 1731956713224Disabling writes for close at 1731956713224Obtaining lock to block concurrent updates at 1731956713228 (+4 ms)Preparing flush snapshotting stores in b38dbac143e9407aa48a95e23085ace3 at 1731956713228Finished memstore snapshotting test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1731956713236 (+8 ms)Flushing stores of test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. at 1731956713238 (+2 ms)Flushing b38dbac143e9407aa48a95e23085ace3/f: creating writer at 1731956713239 (+1 ms)Flushing b38dbac143e9407aa48a95e23085ace3/f: appending metadata at 1731956713284 (+45 ms)Flushing b38dbac143e9407aa48a95e23085ace3/f: closing flushed file at 1731956713288 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17c00edd: reopening flushed file at 1731956713774 (+486 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for b38dbac143e9407aa48a95e23085ace3 in 564ms, sequenceid=103, compaction requested=false at 1731956713793 (+19 ms)Writing region close event to WAL at 1731956713795 (+2 ms)Running coprocessor post-close hooks at 1731956713801 (+6 ms)Closed at 1731956713804 (+3 ms) 2024-11-18T19:05:13,804 WARN [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: b38dbac143e9407aa48a95e23085ace3 to self. 2024-11-18T19:05:13,808 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:13,809 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=CLOSED 2024-11-18T19:05:13,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:13,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-18T19:05:13,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 in 748 msec 2024-11-18T19:05:13,817 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE; state=CLOSED, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=true 2024-11-18T19:05:13,968 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:13,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE because future has completed 2024-11-18T19:05:13,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:14,129 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:14,129 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => b38dbac143e9407aa48a95e23085ace3, NAME => 'test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:14,130 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:14,130 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731956706111 b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,130 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:14,130 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,130 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,132 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,134 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b38dbac143e9407aa48a95e23085ace3 columnFamilyName f 2024-11-18T19:05:14,134 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:14,143 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d 2024-11-18T19:05:14,144 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(327): Store=b38dbac143e9407aa48a95e23085ace3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:14,144 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,146 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b38dbac143e9407aa48a95e23085ace3 columnFamilyName f3 2024-11-18T19:05:14,146 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:14,146 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(327): Store=b38dbac143e9407aa48a95e23085ace3/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:14,148 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,154 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b38dbac143e9407aa48a95e23085ace3 columnFamilyName mob 2024-11-18T19:05:14,159 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:14,159 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(327): Store=b38dbac143e9407aa48a95e23085ace3/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:14,160 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,161 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,162 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,164 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,164 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,165 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1731956706111 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T19:05:14,167 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,168 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened b38dbac143e9407aa48a95e23085ace3; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11786480640, jitterRate=0.0977015495300293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T19:05:14,168 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:14,169 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for b38dbac143e9407aa48a95e23085ace3: Running coprocessor pre-open hook at 1731956714130Writing region info on filesystem at 1731956714130Initializing all the Stores at 1731956714132 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956714132Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956714132Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956714132Cleaning up temporary data from old regions at 1731956714164 (+32 ms)Running coprocessor post-open hooks at 1731956714168 (+4 ms)Region opened successfully at 1731956714169 (+1 ms) 2024-11-18T19:05:14,171 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., pid=24, masterSystemTime=1731956714124 2024-11-18T19:05:14,174 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:14,174 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:14,175 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=OPEN, openSeqNum=107, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:14,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:14,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-18T19:05:14,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 in 208 msec 2024-11-18T19:05:14,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-18T19:05:14,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE in 1.1230 sec 2024-11-18T19:05:14,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-18T19:05:14,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1731956706111 in 1.1370 sec 2024-11-18T19:05:14,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1731956706111 in 1.5780 sec 2024-11-18T19:05:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-18T19:05:14,198 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1731956706111 completed 2024-11-18T19:05:14,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c875cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:14,199 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:14,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:14,200 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:14,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:14,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:14,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b575ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:14,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:14,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:14,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:14,203 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:14,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190c0a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:14,204 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:14,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:14,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:14,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:14,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731956706111', row='row-test-1731956706111-3-0', locateType=CURRENT is [region=test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., hostname=818bff808382,33643,1731956698538, seqNum=107] 2024-11-18T19:05:14,256 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:05:14,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:05:14,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@526d7bf3 2024-11-18T19:05:14,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:05:14,261 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:05:14,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-11-18T19:05:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-18T19:05:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T19:05:14,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 11 msec 2024-11-18T19:05:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T19:05:14,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-18T19:05:14,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-18T19:05:14,404 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:05:14,404 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:14,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-18T19:05:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-18T19:05:14,406 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:05:14,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741849_1025 (size=635) 2024-11-18T19:05:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-18T19:05:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-18T19:05:14,818 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 63e71206ef16a51a40e751c8e3988243, NAME => 'backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:14,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741850_1026 (size=40) 2024-11-18T19:05:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-18T19:05:15,228 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:15,228 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 63e71206ef16a51a40e751c8e3988243, disabling compactions & flushes 2024-11-18T19:05:15,228 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:15,228 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:15,228 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. after waiting 0 ms 2024-11-18T19:05:15,228 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:15,228 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:15,228 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 63e71206ef16a51a40e751c8e3988243: Waiting for close lock at 1731956715228Disabling compacts and flushes for region at 1731956715228Disabling writes for close at 1731956715228Writing region close event to WAL at 1731956715228Closed at 1731956715228 2024-11-18T19:05:15,230 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:05:15,231 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1731956715231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956715231"}]},"ts":"1731956715231"} 2024-11-18T19:05:15,233 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T19:05:15,235 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:05:15,235 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956715235"}]},"ts":"1731956715235"} 2024-11-18T19:05:15,237 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-18T19:05:15,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=63e71206ef16a51a40e751c8e3988243, ASSIGN}] 2024-11-18T19:05:15,239 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=63e71206ef16a51a40e751c8e3988243, ASSIGN 2024-11-18T19:05:15,240 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=63e71206ef16a51a40e751c8e3988243, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:05:15,391 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=63e71206ef16a51a40e751c8e3988243, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:15,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=63e71206ef16a51a40e751c8e3988243, ASSIGN because future has completed 2024-11-18T19:05:15,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63e71206ef16a51a40e751c8e3988243, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-18T19:05:15,552 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:15,553 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 63e71206ef16a51a40e751c8e3988243, NAME => 'backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:15,553 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:15,554 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,554 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:15,554 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,554 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,556 INFO [StoreOpener-63e71206ef16a51a40e751c8e3988243-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,558 INFO [StoreOpener-63e71206ef16a51a40e751c8e3988243-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63e71206ef16a51a40e751c8e3988243 columnFamilyName meta 2024-11-18T19:05:15,558 DEBUG [StoreOpener-63e71206ef16a51a40e751c8e3988243-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:15,559 INFO [StoreOpener-63e71206ef16a51a40e751c8e3988243-1 {}] regionserver.HStore(327): Store=63e71206ef16a51a40e751c8e3988243/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:15,559 INFO [StoreOpener-63e71206ef16a51a40e751c8e3988243-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,561 INFO [StoreOpener-63e71206ef16a51a40e751c8e3988243-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63e71206ef16a51a40e751c8e3988243 columnFamilyName session 2024-11-18T19:05:15,561 DEBUG [StoreOpener-63e71206ef16a51a40e751c8e3988243-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:15,562 INFO [StoreOpener-63e71206ef16a51a40e751c8e3988243-1 {}] regionserver.HStore(327): Store=63e71206ef16a51a40e751c8e3988243/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:15,562 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,563 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,564 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,565 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,565 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,565 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-18T19:05:15,567 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,571 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:05:15,571 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 63e71206ef16a51a40e751c8e3988243; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9446299520, jitterRate=-0.12024480104446411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-18T19:05:15,572 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:15,572 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 63e71206ef16a51a40e751c8e3988243: Running coprocessor pre-open hook at 1731956715554Writing region info on filesystem at 1731956715554Initializing all the Stores at 1731956715556 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956715556Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956715556Cleaning up temporary data from old regions at 1731956715565 (+9 ms)Running coprocessor post-open hooks at 1731956715572 (+7 ms)Region opened successfully at 1731956715572 2024-11-18T19:05:15,573 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., pid=28, masterSystemTime=1731956715547 2024-11-18T19:05:15,576 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:15,576 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:15,577 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=63e71206ef16a51a40e751c8e3988243, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:15,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63e71206ef16a51a40e751c8e3988243, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:15,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-18T19:05:15,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 63e71206ef16a51a40e751c8e3988243, server=818bff808382,33643,1731956698538 in 186 msec 2024-11-18T19:05:15,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-18T19:05:15,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=63e71206ef16a51a40e751c8e3988243, ASSIGN in 346 msec 2024-11-18T19:05:15,588 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:05:15,588 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956715588"}]},"ts":"1731956715588"} 2024-11-18T19:05:15,590 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-18T19:05:15,592 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:05:15,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.1920 sec 2024-11-18T19:05:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-18T19:05:16,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-18T19:05:16,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-18T19:05:16,576 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:05:16,576 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:16,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-18T19:05:16,577 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:05:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T19:05:16,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741851_1027 (size=640) 2024-11-18T19:05:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T19:05:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T19:05:16,989 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e689fbc273d3ab005a435f927e214efb, NAME => 'backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:16,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741852_1028 (size=45) 2024-11-18T19:05:17,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17319567061112 2024-11-18T19:05:17,103 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17319567061112 Metrics about Tables on a single HBase RegionServer 2024-11-18T19:05:17,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-18T19:05:17,104 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-18T19:05:17,105 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1731956706111 2024-11-18T19:05:17,105 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1731956706111 Metrics about Tables on a single HBase RegionServer 2024-11-18T19:05:17,106 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17319567061113 2024-11-18T19:05:17,106 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17319567061113 Metrics about Tables on a single HBase RegionServer 2024-11-18T19:05:17,107 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17319567061111 2024-11-18T19:05:17,107 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17319567061111 Metrics about Tables on a single HBase RegionServer 2024-11-18T19:05:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T19:05:17,399 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:17,399 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing e689fbc273d3ab005a435f927e214efb, disabling compactions & flushes 2024-11-18T19:05:17,399 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:17,399 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:17,399 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. after waiting 0 ms 2024-11-18T19:05:17,399 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:17,399 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:17,400 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for e689fbc273d3ab005a435f927e214efb: Waiting for close lock at 1731956717399Disabling compacts and flushes for region at 1731956717399Disabling writes for close at 1731956717399Writing region close event to WAL at 1731956717399Closed at 1731956717399 2024-11-18T19:05:17,402 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:05:17,402 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731956717402"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956717402"}]},"ts":"1731956717402"} 2024-11-18T19:05:17,405 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T19:05:17,407 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:05:17,407 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956717407"}]},"ts":"1731956717407"} 2024-11-18T19:05:17,409 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-18T19:05:17,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=e689fbc273d3ab005a435f927e214efb, ASSIGN}] 2024-11-18T19:05:17,411 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=e689fbc273d3ab005a435f927e214efb, ASSIGN 2024-11-18T19:05:17,412 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=e689fbc273d3ab005a435f927e214efb, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:05:17,563 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e689fbc273d3ab005a435f927e214efb, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:17,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=e689fbc273d3ab005a435f927e214efb, ASSIGN because future has completed 2024-11-18T19:05:17,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e689fbc273d3ab005a435f927e214efb, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T19:05:17,725 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:17,725 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => e689fbc273d3ab005a435f927e214efb, NAME => 'backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:17,726 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:17,726 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,726 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:17,726 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,726 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,729 INFO [StoreOpener-e689fbc273d3ab005a435f927e214efb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,730 INFO [StoreOpener-e689fbc273d3ab005a435f927e214efb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e689fbc273d3ab005a435f927e214efb columnFamilyName meta 2024-11-18T19:05:17,731 DEBUG [StoreOpener-e689fbc273d3ab005a435f927e214efb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:17,731 INFO [StoreOpener-e689fbc273d3ab005a435f927e214efb-1 {}] regionserver.HStore(327): Store=e689fbc273d3ab005a435f927e214efb/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:17,731 INFO [StoreOpener-e689fbc273d3ab005a435f927e214efb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,733 INFO [StoreOpener-e689fbc273d3ab005a435f927e214efb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e689fbc273d3ab005a435f927e214efb columnFamilyName session 2024-11-18T19:05:17,733 DEBUG [StoreOpener-e689fbc273d3ab005a435f927e214efb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:17,734 INFO [StoreOpener-e689fbc273d3ab005a435f927e214efb-1 {}] regionserver.HStore(327): Store=e689fbc273d3ab005a435f927e214efb/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:17,734 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,735 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system_bulk/e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,736 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system_bulk/e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,737 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,737 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,738 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-18T19:05:17,740 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,743 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system_bulk/e689fbc273d3ab005a435f927e214efb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:05:17,744 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened e689fbc273d3ab005a435f927e214efb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11317152640, jitterRate=0.05399197340011597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-18T19:05:17,744 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:17,745 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for e689fbc273d3ab005a435f927e214efb: Running coprocessor pre-open hook at 1731956717726Writing region info on filesystem at 1731956717727 (+1 ms)Initializing all the Stores at 1731956717728 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956717728Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956717728Cleaning up temporary data from old regions at 1731956717737 (+9 ms)Running coprocessor post-open hooks at 1731956717744 (+7 ms)Region opened successfully at 1731956717745 (+1 ms) 2024-11-18T19:05:17,747 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb., pid=31, masterSystemTime=1731956717719 2024-11-18T19:05:17,750 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:17,750 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:17,751 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e689fbc273d3ab005a435f927e214efb, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:17,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e689fbc273d3ab005a435f927e214efb, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:17,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-18T19:05:17,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e689fbc273d3ab005a435f927e214efb, server=818bff808382,33643,1731956698538 in 187 msec 2024-11-18T19:05:17,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-18T19:05:17,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=e689fbc273d3ab005a435f927e214efb, ASSIGN in 347 msec 2024-11-18T19:05:17,761 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:05:17,761 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956717761"}]},"ts":"1731956717761"} 2024-11-18T19:05:17,764 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-18T19:05:17,765 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:05:17,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.1930 sec 2024-11-18T19:05:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-18T19:05:18,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-18T19:05:18,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:18,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:18,742 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-18T19:05:18,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:05:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-18T19:05:18,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-18T19:05:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956718771 (current time:1731956718771). 2024-11-18T19:05:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:05:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-18T19:05:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3df02f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:18,773 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:18,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:18,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:18,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b5e784a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:18,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:18,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:18,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:18,776 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:18,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc62ba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:18,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:18,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:18,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:18,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:05:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:18,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:05:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-18T19:05:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-18T19:05:18,794 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:05:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T19:05:18,799 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:05:18,815 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:05:18,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741853_1029 (size=52) 2024-11-18T19:05:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T19:05:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T19:05:19,225 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:05:19,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243}] 2024-11-18T19:05:19,234 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:19,308 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-18T19:05:19,380 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-18T19:05:19,381 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-18T19:05:19,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-18T19:05:19,394 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:19,395 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 63e71206ef16a51a40e751c8e3988243 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-18T19:05:19,414 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/86a0cd1ea2f44fae8d9f10d7f1c42889 is 49, key is activesession:/session:c/1731956718760/Put/seqid=0 2024-11-18T19:05:19,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741854_1030 (size=5049) 2024-11-18T19:05:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T19:05:19,822 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/86a0cd1ea2f44fae8d9f10d7f1c42889 2024-11-18T19:05:19,831 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/86a0cd1ea2f44fae8d9f10d7f1c42889 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889 2024-11-18T19:05:19,839 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889, entries=1, sequenceid=5, filesize=4.9 K 2024-11-18T19:05:19,840 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 63e71206ef16a51a40e751c8e3988243 in 446ms, sequenceid=5, compaction requested=false 2024-11-18T19:05:19,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 63e71206ef16a51a40e751c8e3988243: 2024-11-18T19:05:19,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. for snapshot_backup_system completed. 2024-11-18T19:05:19,842 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.' region-info for snapshot=snapshot_backup_system 2024-11-18T19:05:19,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:05:19,848 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-18T19:05:19,848 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889] hfiles 2024-11-18T19:05:19,849 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889 for snapshot=snapshot_backup_system 2024-11-18T19:05:19,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741855_1031 (size=95) 2024-11-18T19:05:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T19:05:20,260 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:20,263 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-18T19:05:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-18T19:05:20,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:20,266 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:20,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-18T19:05:20,270 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:05:20,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 in 1.0390 sec 2024-11-18T19:05:20,272 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:05:20,273 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:05:20,273 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-18T19:05:20,276 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-18T19:05:20,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741856_1032 (size=731) 2024-11-18T19:05:20,694 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:05:20,706 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:05:20,707 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_backup_system 2024-11-18T19:05:20,710 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:05:20,710 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-18T19:05:20,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9210 sec 2024-11-18T19:05:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T19:05:20,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-18T19:05:20,944 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731956714240 started at 1731956720943. 2024-11-18T19:05:20,958 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731956714240 has been started. 2024-11-18T19:05:20,964 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-18T19:05:20,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-18T19:05:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-18T19:05:20,973 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-18T19:05:20,974 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-18T19:05:20,974 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-18T19:05:20,974 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-18T19:05:20,975 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:20,975 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:20,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-18T19:05:20,977 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:20,977 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-18T19:05:20,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:05:20,978 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:20,978 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-18T19:05:20,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:20,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:20,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-18T19:05:20,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:20,979 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-18T19:05:20,981 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-18T19:05:20,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-18T19:05:20,982 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-18T19:05:20,982 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-18T19:05:20,982 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-18T19:05:20,982 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-18T19:05:20,982 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '818bff808382,33643,1731956698538' joining acquired barrier for procedure (rolllog) in zk 2024-11-18T19:05:20,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:20,984 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:20,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:20,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:20,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:20,985 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-18T19:05:20,985 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-18T19:05:20,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:20,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:20,987 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:20,987 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:20,988 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:20,988 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '818bff808382,33643,1731956698538' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-18T19:05:20,988 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7e10ee7e[Count = 0] remaining members to acquire global barrier 2024-11-18T19:05:20,989 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-18T19:05:20,989 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:20,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:20,990 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:20,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:20,991 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-18T19:05:20,991 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:20,991 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-18T19:05:20,991 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-18T19:05:20,992 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 818bff808382,33643,1731956698538 2024-11-18T19:05:20,998 INFO [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731956701226 highest: 1731956707820 on 818bff808382,33643,1731956698538 2024-11-18T19:05:20,998 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1:(num 1731956707820) roll requested 2024-11-18T19:05:21,021 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999, exclude list is [], retry=0 2024-11-18T19:05:21,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:21,027 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956707820 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 2024-11-18T19:05:21,028 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:21,028 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956707820 is not closed yet, will try archiving it next time 2024-11-18T19:05:21,028 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.meta:.meta(num 1731956701701) roll requested 2024-11-18T19:05:21,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741840_1016 (size=4756) 2024-11-18T19:05:21,042 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta, exclude list is [], retry=0 2024-11-18T19:05:21,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:21,048 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956701701.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta 2024-11-18T19:05:21,048 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:21,049 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956701701.meta is not closed yet, will try archiving it next time 2024-11-18T19:05:21,049 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0:(num 1731956701226) roll requested 2024-11-18T19:05:21,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741834_1010 (size=12055) 2024-11-18T19:05:21,063 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049, exclude list is [], retry=0 2024-11-18T19:05:21,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:21,068 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956701226 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 2024-11-18T19:05:21,068 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:21,068 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956701226 is not closed yet, will try archiving it next time 2024-11-18T19:05:21,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741833_1009 (size=1034) 2024-11-18T19:05:21,072 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956701226 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956701226 2024-11-18T19:05:21,079 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 81 2024-11-18T19:05:21,079 INFO [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731956721049 on 818bff808382,33643,1731956698538 2024-11-18T19:05:21,079 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:05:21,079 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:05:21,079 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@50660453 2024-11-18T19:05:21,080 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:05:21,081 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35287, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-18T19:05:21,082 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:21,082 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:21,083 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:21,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57659, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-18T19:05:21,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:21,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:21,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:45363/backupUT\x00', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:05:21,111 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-18T19:05:21,111 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '818bff808382,33643,1731956698538' in zk 2024-11-18T19:05:21,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:21,113 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-18T19:05:21,113 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-18T19:05:21,114 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:21,114 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:21,114 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-18T19:05:21,114 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:21,114 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:21,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:21,115 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:21,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:21,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:21,116 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:21,117 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:21,118 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '818bff808382,33643,1731956698538': 2024-11-18T19:05:21,118 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-18T19:05:21,118 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-18T19:05:21,118 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-18T19:05:21,118 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '818bff808382,33643,1731956698538' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-18T19:05:21,118 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-18T19:05:21,118 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-18T19:05:21,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:21,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:21,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:21,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:21,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:21,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-18T19:05:21,121 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:21,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:21,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:21,122 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:21,122 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-18T19:05:21,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:21,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:05:21,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:21,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:21,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:21,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:21,124 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:21,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:21,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:21,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:21,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:21,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:21,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:21,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-18T19:05:21,131 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-18T19:05:21,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:21,132 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:05:21,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:21,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:21,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:21,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-18T19:05:21,132 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-18T19:05:21,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-18T19:05:21,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-18T19:05:21,133 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-18T19:05:21,133 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:05:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-18T19:05:21,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } 2024-11-18T19:05:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956721149 (current time:1731956721149). 2024-11-18T19:05:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:05:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731956721148_default_test-1731956706111 VERSION not specified, setting to 2 2024-11-18T19:05:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a164300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:21,151 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:21,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:21,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:21,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1dcf15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:21,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:21,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:21,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:21,153 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:21,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e98bcf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:21,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:21,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:21,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:21,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:05:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:21,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:05:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } 2024-11-18T19:05:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-18T19:05:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T19:05:21,160 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:05:21,162 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:05:21,165 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:05:21,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741860_1036 (size=84) 2024-11-18T19:05:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T19:05:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T19:05:21,573 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:05:21,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b38dbac143e9407aa48a95e23085ace3}] 2024-11-18T19:05:21,575 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:21,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-18T19:05:21,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:21,728 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing b38dbac143e9407aa48a95e23085ace3 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-18T19:05:21,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f3/b7ecceb2351a40c6895d36dc6ee8cc50 is 58, key is row-test-1731956706111-3-0/f3:q1/1731956714211/Put/seqid=0 2024-11-18T19:05:21,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741861_1037 (size=5387) 2024-11-18T19:05:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T19:05:22,154 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f3/b7ecceb2351a40c6895d36dc6ee8cc50 2024-11-18T19:05:22,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741862_1038 (size=4469) 2024-11-18T19:05:22,200 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/mob/279d7b241f8c4edd9070df580f246dc0 is 59, key is row-test-1731956706111-3-0/mob:q1/1731956714226/Put/seqid=0 2024-11-18T19:05:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741863_1039 (size=5484) 2024-11-18T19:05:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T19:05:22,608 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/mob/279d7b241f8c4edd9070df580f246dc0 2024-11-18T19:05:22,619 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f3/b7ecceb2351a40c6895d36dc6ee8cc50 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 2024-11-18T19:05:22,628 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50, entries=6, sequenceid=121, filesize=5.3 K 2024-11-18T19:05:22,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/mob/279d7b241f8c4edd9070df580f246dc0 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 2024-11-18T19:05:22,638 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0, entries=6, sequenceid=121, filesize=5.4 K 2024-11-18T19:05:22,639 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for b38dbac143e9407aa48a95e23085ace3 in 911ms, sequenceid=121, compaction requested=false 2024-11-18T19:05:22,639 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for b38dbac143e9407aa48a95e23085ace3: 2024-11-18T19:05:22,639 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. for snapshot_1731956721148_default_test-1731956706111 completed. 2024-11-18T19:05:22,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.' region-info for snapshot=snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:22,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:05:22,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d] hfiles 2024-11-18T19:05:22,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d for snapshot=snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:22,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50] hfiles 2024-11-18T19:05:22,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 for snapshot=snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:22,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0] hfiles 2024-11-18T19:05:22,640 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 for snapshot=snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:22,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741864_1040 (size=185) 2024-11-18T19:05:23,047 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:23,047 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-18T19:05:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-18T19:05:23,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731956721148_default_test-1731956706111 on region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:23,049 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:23,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-18T19:05:23,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure b38dbac143e9407aa48a95e23085ace3 in 1.4760 sec 2024-11-18T19:05:23,052 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:05:23,053 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:05:23,056 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-18T19:05:23,056 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-18T19:05:23,056 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:23,057 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-18T19:05:23,057 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:23,058 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-18T19:05:23,058 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:23,058 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-18T19:05:23,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741865_1041 (size=49) 2024-11-18T19:05:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T19:05:23,468 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:05:23,468 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:23,469 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:23,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741866_1042 (size=1146) 2024-11-18T19:05:23,884 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:05:23,895 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:05:23,895 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956721148_default_test-1731956706111 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:23,897 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:05:23,898 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-18T19:05:23,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731956721148_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } in 2.7400 sec 2024-11-18T19:05:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-18T19:05:25,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1731956706111 completed 2024-11-18T19:05:25,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } 2024-11-18T19:05:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956725323 (current time:1731956725323). 2024-11-18T19:05:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:05:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731956725322_ns2_test-17319567061111 VERSION not specified, setting to 2 2024-11-18T19:05:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cfce7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:25,325 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:25,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:25,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:25,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f8a180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:25,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:25,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:25,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:25,327 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:25,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3894ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:25,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:25,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:25,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:25,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:05:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:25,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:05:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } 2024-11-18T19:05:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-18T19:05:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T19:05:25,336 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:05:25,338 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:05:25,342 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:05:25,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741867_1043 (size=86) 2024-11-18T19:05:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T19:05:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T19:05:25,750 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:05:25,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb}] 2024-11-18T19:05:25,752 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:25,754 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T19:05:25,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-18T19:05:25,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:25,905 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing d144bc0c998ecf0ed9ce62cb631ee3eb 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-18T19:05:25,924 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/.tmp/f/2bf819358c514f5bb172f1c243472408 is 37, key is row10/f:q1/1731956709595/Put/seqid=0 2024-11-18T19:05:25,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741868_1044 (size=8413) 2024-11-18T19:05:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T19:05:26,331 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/.tmp/f/2bf819358c514f5bb172f1c243472408 2024-11-18T19:05:26,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/.tmp/f/2bf819358c514f5bb172f1c243472408 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 2024-11-18T19:05:26,350 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408, entries=99, sequenceid=103, filesize=8.2 K 2024-11-18T19:05:26,352 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for d144bc0c998ecf0ed9ce62cb631ee3eb in 447ms, sequenceid=103, compaction requested=false 2024-11-18T19:05:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for d144bc0c998ecf0ed9ce62cb631ee3eb: 2024-11-18T19:05:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. for snapshot_1731956725322_ns2_test-17319567061111 completed. 2024-11-18T19:05:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb.' region-info for snapshot=snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:05:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408] hfiles 2024-11-18T19:05:26,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 for snapshot=snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:26,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741869_1045 (size=91) 2024-11-18T19:05:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T19:05:26,759 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:26,759 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-18T19:05:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-18T19:05:26,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731956725322_ns2_test-17319567061111 on region d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:26,760 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:26,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-18T19:05:26,764 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:05:26,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb in 1.0120 sec 2024-11-18T19:05:26,765 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:05:26,766 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:05:26,766 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:26,767 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:26,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741870_1046 (size=456) 2024-11-18T19:05:27,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-18T19:05:27,103 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-18T19:05:27,179 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:05:27,187 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:05:27,188 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956725322_ns2_test-17319567061111 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:27,190 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:05:27,190 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-18T19:05:27,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731956725322_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } in 1.8560 sec 2024-11-18T19:05:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T19:05:27,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17319567061111 completed 2024-11-18T19:05:27,483 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1731956714240 2024-11-18T19:05:27,483 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-18T19:05:27,485 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-18T19:05:27,487 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731956714240_test-1731956706111 2024-11-18T19:05:27,487 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731956721148_default_test-1731956706111 to hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/ 2024-11-18T19:05:27,492 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-18T19:05:27,530 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:45363, inputRoot=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:27,530 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1308346821_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/.hbase-snapshot/.tmp/snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:27,533 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-18T19:05:27,541 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956721148_default_test-1731956706111 to hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/.hbase-snapshot/.tmp/snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:27,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741871_1047 (size=84) 2024-11-18T19:05:27,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741872_1048 (size=1146) 2024-11-18T19:05:27,959 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:27,960 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:27,960 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,002 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-5370978902201698631.jar 2024-11-18T19:05:29,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,004 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,004 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,004 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,005 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,005 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,005 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,006 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:29,006 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:05:29,006 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:05:29,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:05:29,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:05:29,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:05:29,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:05:29,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:05:29,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:05:29,009 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:05:29,009 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:05:29,009 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:05:29,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:05:29,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:05:29,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:05:29,012 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:05:29,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:05:29,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:05:29,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:05:29,086 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:05:29,121 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731956721148_default_test-1731956706111' hfile list 2024-11-18T19:05:29,128 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=18.8 K 2024-11-18T19:05:29,424 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local196409342_0001_2276645b-3d7d-4b77-9dce-7663685d39a0/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:05:29,425 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local196409342_0001_2276645b-3d7d-4b77-9dce-7663685d39a0/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:05:29,630 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-18T19:05:29,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741873_1049 (size=8413) 2024-11-18T19:05:30,142 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d output=hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d 2024-11-18T19:05:30,145 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.016M/sec 2024-11-18T19:05:30,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741874_1050 (size=5484) 2024-11-18T19:05:30,597 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 output=hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 2024-11-18T19:05:30,597 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-18T19:05:30,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741875_1051 (size=5387) 2024-11-18T19:05:31,070 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 output=hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 2024-11-18T19:05:31,070 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-11-18T19:05:31,534 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-18T19:05:31,535 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-18T19:05:31,546 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:31,546 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731956721148_default_test-1731956706111 finished. 2024-11-18T19:05:31,547 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731956714240_ns2:test-17319567061111 2024-11-18T19:05:31,547 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731956725322_ns2_test-17319567061111 to hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/ 2024-11-18T19:05:31,547 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-18T19:05:31,589 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:45363, inputRoot=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:05:31,589 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1308346821_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/.hbase-snapshot/.tmp/snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:31,591 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-18T19:05:31,598 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956725322_ns2_test-17319567061111 to hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/.hbase-snapshot/.tmp/snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:31,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741876_1052 (size=86) 2024-11-18T19:05:31,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741877_1053 (size=456) 2024-11-18T19:05:32,014 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:32,014 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:32,015 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,033 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-17951643344837699439.jar 2024-11-18T19:05:33,034 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,034 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,034 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,034 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,035 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,035 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,035 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:33,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:05:33,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:05:33,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:05:33,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:05:33,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:05:33,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:05:33,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:05:33,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:05:33,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:05:33,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:05:33,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:05:33,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:05:33,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:05:33,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:05:33,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:05:33,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:05:33,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:05:33,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:05:33,085 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:05:33,116 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731956725322_ns2_test-17319567061111' hfile list 2024-11-18T19:05:33,119 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=8.2 K 2024-11-18T19:05:33,235 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local2027183494_0002_369e6732-2506-4aae-83b4-709561584aa4/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:05:33,235 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local2027183494_0002_369e6732-2506-4aae-83b4-709561584aa4/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:05:33,314 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-18T19:05:33,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741878_1054 (size=8413) 2024-11-18T19:05:33,808 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 output=hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 2024-11-18T19:05:33,808 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.017M/sec 2024-11-18T19:05:34,280 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-18T19:05:34,282 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-18T19:05:34,290 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:34,290 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731956725322_ns2_test-17319567061111 finished. 2024-11-18T19:05:34,291 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1731956706111 2024-11-18T19:05:34,291 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17319567061111 2024-11-18T19:05:34,351 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731956714240 2024-11-18T19:05:34,351 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-18T19:05:34,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741879_1055 (size=130) 2024-11-18T19:05:34,762 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45363/backupUT/backup_1731956714240/.backup.manifest 2024-11-18T19:05:34,765 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731956714240 finished: type=FULL,tablelist=test-1731956706111;ns2:test-17319567061111,targetRootDir=hdfs://localhost:45363/backupUT,startts=1731956720943,completets=1731956734348,bytescopied=0 2024-11-18T19:05:34,765 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-18T19:05:34,765 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:34,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731956721148_default_test-1731956706111" type: DISABLED 2024-11-18T19:05:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731956721148_default_test-1731956706111 2024-11-18T19:05:34,777 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731956721148_default_test-1731956706111 for backup backup_1731956714240 succeeded. 2024-11-18T19:05:34,777 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:34,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731956725322_ns2_test-17319567061111" type: DISABLED 2024-11-18T19:05:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731956725322_ns2_test-17319567061111 2024-11-18T19:05:34,781 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731956725322_ns2_test-17319567061111 for backup backup_1731956714240 succeeded. 2024-11-18T19:05:34,782 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-18T19:05:34,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-18T19:05:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-18T19:05:34,797 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-18T19:05:34,803 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-18T19:05:34,825 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731956714240 completed. 2024-11-18T19:05:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1731956714240', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:05:35,035 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1731956706111 2024-11-18T19:05:35,056 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1731956706111 2024-11-18T19:05:35,247 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1731956706111 to Mob enabled CF 2024-11-18T19:05:35,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17319567061111', row='row-t20', locateType=CURRENT is [region=ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:05:35,287 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17319567061111 2024-11-18T19:05:35,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:35,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] assignment.SplitTableRegionProcedure(210): test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. splitKey isn't explicitly specified, will try to find a best split key from RS 818bff808382,33643,1731956698538 2024-11-18T19:05:35,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d because midkey is the same as first or last row 2024-11-18T19:05:35,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 because midkey is the same as first or last row 2024-11-18T19:05:35,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 because midkey is the same as first or last row 2024-11-18T19:05:35,303 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.HRegion(2902): Flushing b38dbac143e9407aa48a95e23085ace3 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-18T19:05:35,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f/b77fe233b3544d71b762e205f6df7bad is 59, key is row-test-1731956706111-1-10/f:q1/1731956734874/Put/seqid=0 2024-11-18T19:05:35,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741880_1056 (size=10796) 2024-11-18T19:05:35,731 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f/b77fe233b3544d71b762e205f6df7bad 2024-11-18T19:05:35,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3 is 63, key is row-test-1731956706111-3-100/mob:q1/1731956735228/Put/seqid=0 2024-11-18T19:05:35,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741881_1057 (size=5591) 2024-11-18T19:05:36,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:36,162 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1/mob/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:36,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/mob/c51bc94ac86f4f448c46a9da09c3d19c, store: [table=test-1731956706111 family=mob region=b38dbac143e9407aa48a95e23085ace3] 2024-11-18T19:05:36,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/mob/c51bc94ac86f4f448c46a9da09c3d19c is 192, key is row-test-1731956706111-3-100/mob:q1/1731956735228/Put/seqid=0 2024-11-18T19:05:36,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741882_1058 (size=13760) 2024-11-18T19:05:36,573 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/mob/c51bc94ac86f4f448c46a9da09c3d19c 2024-11-18T19:05:36,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f/b77fe233b3544d71b762e205f6df7bad as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad 2024-11-18T19:05:36,590 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad, entries=99, sequenceid=334, filesize=10.5 K 2024-11-18T19:05:36,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/mob/c51bc94ac86f4f448c46a9da09c3d19c as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c 2024-11-18T19:05:36,598 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c, entries=111, sequenceid=334, filesize=13.4 K 2024-11-18T19:05:36,599 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for b38dbac143e9407aa48a95e23085ace3 in 1297ms, sequenceid=334, compaction requested=false 2024-11-18T19:05:36,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.HRegion(2603): Flush status journal for b38dbac143e9407aa48a95e23085ace3: 2024-11-18T19:05:36,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad because midkey is the same as first or last row 2024-11-18T19:05:36,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 because midkey is the same as first or last row 2024-11-18T19:05:36,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c because midkey is the same as first or last row 2024-11-18T19:05:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=818bff808382,33643,1731956698538 2024-11-18T19:05:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T19:05:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.CallRunner(138): callId: 303 service: MasterService methodName: SplitRegion size: 100 connection: 172.17.0.2:57366 deadline: 1731956795295, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-18T19:05:36,609 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:36,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1731956706111 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:36,609 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T19:05:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731956706111,, stopping at row=test-1731956706111 ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,623 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731956706111,, stopping at row=test-1731956706111 ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,633 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1346 2024-11-18T19:05:36,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,677 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-18T19:05:36,682 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731956736633 2024-11-18T19:05:36,690 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731956714240} 2024-11-18T19:05:36,690 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-18T19:05:36,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:36,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-18T19:05:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956736739 (current time:1731956736739). 2024-11-18T19:05:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:05:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-18T19:05:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a6a0989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:36,741 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:36,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:36,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:36,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375fef87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:36,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:36,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:36,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:36,744 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:36,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636ea477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:36,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:36,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:36,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:36,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:05:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:36,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:05:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-18T19:05:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-18T19:05:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-18T19:05:36,752 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:05:36,753 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:05:36,755 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:05:36,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741883_1059 (size=52) 2024-11-18T19:05:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-18T19:05:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-18T19:05:37,163 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:05:37,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243}] 2024-11-18T19:05:37,165 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:37,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-18T19:05:37,317 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:37,318 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 63e71206ef16a51a40e751c8e3988243 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-18T19:05:37,335 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4519732d1ea9401aaac2675ef7be1464 is 158, key is trslm:hdfs://localhost:45363/backupUT\x00ns2:test-17319567061111/meta:log-roll-map/1731956734336/Put/seqid=0 2024-11-18T19:05:37,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741884_1060 (size=5914) 2024-11-18T19:05:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-18T19:05:37,741 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4519732d1ea9401aaac2675ef7be1464 2024-11-18T19:05:37,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/82b014184a204d09b4c21c43a476139b is 433, key is session:backup_1731956714240/session:context/1731956734801/Put/seqid=0 2024-11-18T19:05:37,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741885_1061 (size=5535) 2024-11-18T19:05:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-18T19:05:38,157 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/82b014184a204d09b4c21c43a476139b 2024-11-18T19:05:38,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4519732d1ea9401aaac2675ef7be1464 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4519732d1ea9401aaac2675ef7be1464 2024-11-18T19:05:38,171 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4519732d1ea9401aaac2675ef7be1464, entries=6, sequenceid=17, filesize=5.8 K 2024-11-18T19:05:38,172 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/82b014184a204d09b4c21c43a476139b as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/82b014184a204d09b4c21c43a476139b 2024-11-18T19:05:38,178 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/82b014184a204d09b4c21c43a476139b, entries=2, sequenceid=17, filesize=5.4 K 2024-11-18T19:05:38,179 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 63e71206ef16a51a40e751c8e3988243 in 861ms, sequenceid=17, compaction requested=false 2024-11-18T19:05:38,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 63e71206ef16a51a40e751c8e3988243: 2024-11-18T19:05:38,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. for snapshot_backup_system completed. 2024-11-18T19:05:38,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.' region-info for snapshot=snapshot_backup_system 2024-11-18T19:05:38,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:05:38,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4519732d1ea9401aaac2675ef7be1464] hfiles 2024-11-18T19:05:38,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4519732d1ea9401aaac2675ef7be1464 for snapshot=snapshot_backup_system 2024-11-18T19:05:38,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/82b014184a204d09b4c21c43a476139b] hfiles 2024-11-18T19:05:38,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889 for snapshot=snapshot_backup_system 2024-11-18T19:05:38,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/82b014184a204d09b4c21c43a476139b for snapshot=snapshot_backup_system 2024-11-18T19:05:38,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741886_1062 (size=173) 2024-11-18T19:05:38,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:38,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-18T19:05:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-18T19:05:38,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:38,587 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:38,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-18T19:05:38,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 in 1.4250 sec 2024-11-18T19:05:38,590 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:05:38,591 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:05:38,592 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:05:38,592 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-18T19:05:38,593 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-18T19:05:38,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741887_1063 (size=810) 2024-11-18T19:05:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-18T19:05:39,005 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:05:39,014 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:05:39,014 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_backup_system 2024-11-18T19:05:39,016 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:05:39,016 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-18T19:05:39,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2670 sec 2024-11-18T19:05:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-18T19:05:40,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-18T19:05:40,933 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731956736633 started at 1731956740933. 2024-11-18T19:05:40,935 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731956736633 has been started. 2024-11-18T19:05:40,940 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [test-1731956706111, ns2:test-17319567061111] 2024-11-18T19:05:40,944 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1731956707820for backupID backup_1731956736633 2024-11-18T19:05:40,944 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-18T19:05:40,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-18T19:05:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-18T19:05:40,945 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-18T19:05:40,945 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-18T19:05:40,945 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-18T19:05:40,945 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-18T19:05:40,946 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:40,946 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:40,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-18T19:05:40,951 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:40,952 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-18T19:05:40,952 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:05:40,952 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:40,952 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-18T19:05:40,952 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:40,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:40,953 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-18T19:05:40,953 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:40,953 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-18T19:05:40,954 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-18T19:05:40,954 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-18T19:05:40,954 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-18T19:05:40,954 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-18T19:05:40,954 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-18T19:05:40,954 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-18T19:05:40,954 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '818bff808382,33643,1731956698538' joining acquired barrier for procedure (rolllog) in zk 2024-11-18T19:05:40,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:40,956 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:40,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:40,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:40,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:40,956 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-18T19:05:40,956 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-18T19:05:40,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:40,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:40,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:40,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:40,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:40,958 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '818bff808382,33643,1731956698538' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-18T19:05:40,959 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5b59316e[Count = 0] remaining members to acquire global barrier 2024-11-18T19:05:40,959 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-18T19:05:40,959 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:40,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:40,960 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:40,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:40,960 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-18T19:05:40,960 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:40,960 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-18T19:05:40,961 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-18T19:05:40,961 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 818bff808382,33643,1731956698538 2024-11-18T19:05:40,961 INFO [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731956721049 highest: 1731956721049 on 818bff808382,33643,1731956698538 2024-11-18T19:05:40,961 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1:(num 1731956720999) roll requested 2024-11-18T19:05:40,984 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956740961, exclude list is [], retry=0 2024-11-18T19:05:40,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:40,990 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956740961 2024-11-18T19:05:40,990 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:40,990 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 is not closed yet, will try archiving it next time 2024-11-18T19:05:40,991 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.meta:.meta(num 1731956721028) roll requested 2024-11-18T19:05:40,991 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956707820 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956707820 2024-11-18T19:05:40,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741857_1033 (size=30490) 2024-11-18T19:05:40,992 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 not finished, retry = 0 2024-11-18T19:05:41,005 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956740991.meta, exclude list is [], retry=0 2024-11-18T19:05:41,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:41,009 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956740991.meta 2024-11-18T19:05:41,009 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:41,009 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta is not closed yet, will try archiving it next time 2024-11-18T19:05:41,009 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0:(num 1731956721049) roll requested 2024-11-18T19:05:41,011 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta not finished, retry = 0 2024-11-18T19:05:41,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741858_1034 (size=93) 2024-11-18T19:05:41,023 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956741010, exclude list is [], retry=0 2024-11-18T19:05:41,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:41,028 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956741010 2024-11-18T19:05:41,028 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:41,028 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 is not closed yet, will try archiving it next time 2024-11-18T19:05:41,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741859_1035 (size=1198) 2024-11-18T19:05:41,030 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 not finished, retry = 0 2024-11-18T19:05:41,041 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 80 2024-11-18T19:05:41,041 INFO [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731956741010 on 818bff808382,33643,1731956698538 2024-11-18T19:05:41,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:41,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:41,064 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-18T19:05:41,064 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '818bff808382,33643,1731956698538' in zk 2024-11-18T19:05:41,066 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-18T19:05:41,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:41,066 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-18T19:05:41,066 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-18T19:05:41,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:41,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:41,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:41,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:41,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:41,068 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:41,068 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:41,069 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:41,069 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:41,070 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:41,071 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '818bff808382,33643,1731956698538': 2024-11-18T19:05:41,071 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '818bff808382,33643,1731956698538' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-18T19:05:41,071 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-18T19:05:41,071 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-18T19:05:41,071 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-18T19:05:41,071 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-18T19:05:41,071 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-18T19:05:41,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:41,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:41,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-18T19:05:41,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:41,074 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:41,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:41,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:41,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:41,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:41,074 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-18T19:05:41,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:05:41,075 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:41,075 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:41,075 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:41,076 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:41,077 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:41,077 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:41,078 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:41,078 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:41,079 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:41,079 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:41,082 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-18T19:05:41,082 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:41,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:41,082 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-18T19:05:41,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-18T19:05:41,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-18T19:05:41,083 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-18T19:05:41,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:05:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-18T19:05:41,096 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {818bff808382:33643=1731956707820} newestTimestamps: {818bff808382:33643=1731956721049} 2024-11-18T19:05:41,099 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 818bff808382%2C33643%2C1731956698538.meta.1731956701701.meta 2024-11-18T19:05:41,099 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta 2024-11-18T19:05:41,099 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 818bff808382%2C33643%2C1731956698538.meta.1731956740991.meta 2024-11-18T19:05:41,099 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 2024-11-18T19:05:41,099 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956741010 2024-11-18T19:05:41,099 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 2024-11-18T19:05:41,099 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956740961 2024-11-18T19:05:41,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741891_1067 (size=908) 2024-11-18T19:05:41,116 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta 2024-11-18T19:05:41,515 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1731956706111 target: hdfs://localhost:45363/backupUT/backup_1731956736633/default/test-1731956706111 descriptor: 'test-1731956706111', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:41,515 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-18T19:05:41,518 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1731956706111 2024-11-18T19:05:41,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741892_1068 (size=53) 2024-11-18T19:05:41,924 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1731956706111 2024-11-18T19:05:41,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741893_1069 (size=364) 2024-11-18T19:05:41,937 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17319567061111 target: hdfs://localhost:45363/backupUT/backup_1731956736633/ns2/test-17319567061111 descriptor: 'ns2:test-17319567061111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:41,937 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-18T19:05:41,940 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17319567061111 2024-11-18T19:05:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741894_1070 (size=50) 2024-11-18T19:05:41,972 DEBUG [master/818bff808382:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731956706111 2024-11-18T19:05:41,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741895_1071 (size=950) 2024-11-18T19:05:42,348 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17319567061111 2024-11-18T19:05:42,358 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:45363/backupUT/.tmp/backup_1731956736633 from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049;hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 2024-11-18T19:05:42,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d74c512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:42,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:42,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:42,361 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:42,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:42,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:42,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6afc8c05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:42,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:42,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:42,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:42,362 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:42,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6734a6d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:42,364 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:05:42,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:05:42,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56d61db1 2024-11-18T19:05:42,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:05:42,366 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:05:42,372 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-18T19:05:42,372 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1731956706111 2024-11-18T19:05:42,372 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731956706111,, stopping at row=test-1731956706111 ,, for max=2147483647 with caching=100 2024-11-18T19:05:42,373 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:42,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:42,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:42,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:42,378 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1731956706111: test-1731956706111; 2024-11-18T19:05:42,379 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17319567061111 2024-11-18T19:05:42,379 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17319567061111,, stopping at row=ns2:test-17319567061111 ,, for max=2147483647 with caching=100 2024-11-18T19:05:42,381 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17319567061111: ns2:test-17319567061111; 2024-11-18T19:05:42,381 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-18T19:05:42,381 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/partitions_1a105869-1228-4864-868c-7cf332787c82 2024-11-18T19:05:42,387 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/.tabledesc/.tableinfo.0000000002.908 2024-11-18T19:05:42,387 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/.tabledesc/.tableinfo.0000000003.950 2024-11-18T19:05:42,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731956706111 in 420 msec 2024-11-18T19:05:42,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741896_1072 (size=192) 2024-11-18T19:05:42,807 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:42,808 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:42,808 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-7655969752257116372.jar 2024-11-18T19:05:43,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:05:43,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:05:43,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:05:43,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:05:43,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:05:43,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:05:43,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:05:43,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:05:43,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:05:43,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:05:43,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:05:43,802 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:05:43,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:05:43,804 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: default:test-1731956706111,ns2:test-17319567061111 2024-11-18T19:05:43,805 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:05:43,805 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:05:43,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:43,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:43,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:43,805 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:05:44,791 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-8117154364454406382.jar 2024-11-18T19:05:44,806 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:05:44,834 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 for WAL files 2024-11-18T19:05:44,836 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 2024-11-18T19:05:44,836 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 for WAL files 2024-11-18T19:05:44,837 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 2024-11-18T19:05:44,947 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1267246893_0003_991c69f6-6c43-4fb3-abe0-c3a28fb7f6bc/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:05:44,947 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1267246893_0003_991c69f6-6c43-4fb3-abe0-c3a28fb7f6bc/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:05:45,062 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 for hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 (-9223372036854775808:9223372036854775807) length:30490 2024-11-18T19:05:45,083 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-18T19:05:45,083 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-18T19:05:45,137 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 for hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 (-9223372036854775808:9223372036854775807) length:1198 2024-11-18T19:05:45,142 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-18T19:05:45,142 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-18T19:05:45,241 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479fcbf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,241 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:45,241 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:45,242 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:45,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:45,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:45,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0d5ff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:45,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:45,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:45,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd493cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,246 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:45,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:45,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:45,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:45,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17319567061111', row='row-t20', locateType=CURRENT is [region=ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:05:45,251 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-18T19:05:45,251 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:05:45,251 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,252 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:45,252 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-18T19:05:45,252 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:05:45,259 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/backupUT/.tmp/backup_1731956736633/_temporary/0/_temporary/attempt_local1267246893_0003_r_000000_0/ns2/test-17319567061111/f/ffcbb0ed686348c09e64a939190e8298 is 38, key is row-t20/f:q1/1731956735270/Put/seqid=0 2024-11-18T19:05:45,261 WARN [Thread-949 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:05:45,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741897_1073 (size=5275) 2024-11-18T19:05:45,697 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0073f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,697 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:45,697 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:45,698 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:45,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:45,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:45,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69e6a407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:45,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:45,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,700 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:45,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d39902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,701 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:45,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:45,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:45,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:45,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731956706111', row='row-test-1731956706111-1-0', locateType=CURRENT is [region=test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., hostname=818bff808382,33643,1731956698538, seqNum=107] 2024-11-18T19:05:45,705 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-18T19:05:45,705 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:05:45,705 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,706 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:45,706 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731956706111-1-0] 2024-11-18T19:05:45,706 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:05:45,717 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453bb3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,717 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:45,717 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:45,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:45,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:45,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:45,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45572af2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:45,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:45,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:45,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d7ddb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:45,721 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:45,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:45,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:45,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:45,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731956706111', row='row-test-1731956706111-3-0', locateType=CURRENT is [region=test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., hostname=818bff808382,33643,1731956698538, seqNum=107] 2024-11-18T19:05:45,725 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-18T19:05:45,725 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:05:45,725 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,725 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:45,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:45,725 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731956706111-3-0] 2024-11-18T19:05:45,725 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:05:45,733 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/backupUT/.tmp/backup_1731956736633/_temporary/0/_temporary/attempt_local1267246893_0003_r_000001_0/default/test-1731956706111/f/e58296b917d143bcb4b7d20e03f96862 is 59, key is row-test-1731956706111-1-10/f:q1/1731956734874/Put/seqid=0 2024-11-18T19:05:45,735 WARN [Thread-961 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:05:45,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741898_1074 (size=10770) 2024-11-18T19:05:46,138 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/backupUT/.tmp/backup_1731956736633/_temporary/0/_temporary/attempt_local1267246893_0003_r_000001_0/default/test-1731956706111/mob/2f0f902773c043abaf19b66b67e60d5a is 63, key is row-test-1731956706111-3-100/mob:q1/1731956735228/Put/seqid=0 2024-11-18T19:05:46,140 WARN [Thread-963 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:05:46,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741899_1075 (size=11811) 2024-11-18T19:05:47,002 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:45363/backupUT 2024-11-18T19:05:47,002 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1731956736633 2024-11-18T19:05:47,002 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-18T19:05:47,023 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:45363/backupUT/.tmp/backup_1731956736633, hdfs://localhost:45363/backupUT] 2024-11-18T19:05:47,102 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:05:47,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741900_1076 (size=5275) 2024-11-18T19:05:47,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741901_1077 (size=11811) 2024-11-18T19:05:48,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741902_1078 (size=10770) 2024-11-18T19:05:48,725 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-18T19:05:48,729 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1731956736633 2024-11-18T19:05:48,729 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-11-18T19:05:48,729 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local549024296_0004 completed: true true 2024-11-18T19:05:48,730 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3824359 FILE: Number of bytes written=6206623 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231425 HDFS: Number of bytes written=2433143 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=248 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-11-18T19:05:48,730 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:45363/backupUT/.tmp/backup_1731956736633 to hdfs://localhost:45363/backupUT finished. 2024-11-18T19:05:48,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:test-1731956706111:', locateType=CURRENT is [region=backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:05:48,747 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731956736633 2024-11-18T19:05:48,750 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731956714240} 2024-11-18T19:05:48,750 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-18T19:05:48,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741903_1079 (size=384) 2024-11-18T19:05:49,156 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45363/backupUT/backup_1731956736633/.backup.manifest 2024-11-18T19:05:49,156 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731956736633 finished: type=INCREMENTAL,tablelist=test-1731956706111;ns2:test-17319567061111,targetRootDir=hdfs://localhost:45363/backupUT,startts=1731956740933,completets=1731956748747,bytescopied=0 2024-11-18T19:05:49,156 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-18T19:05:49,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-18T19:05:49,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-18T19:05:49,167 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-18T19:05:49,170 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-18T19:05:49,175 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731956736633 completed. 2024-11-18T19:05:49,175 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-18T19:05:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:49,195 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45363/backupUT/backup_1731956736633/.backup.manifest 2024-11-18T19:05:49,199 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731956736633/.backup.manifest 2024-11-18T19:05:49,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1731956706111 from 'test-1731956706111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1731956706111', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:05:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1731956706111 2024-11-18T19:05:49,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741904_1080 (size=908) 2024-11-18T19:05:49,618 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/.tabledesc/.tableinfo.0000000003.950 2024-11-18T19:05:49,618 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/.tabledesc/.tableinfo.0000000004.908 2024-11-18T19:05:49,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1731956706111}] 2024-11-18T19:05:49,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE}] 2024-11-18T19:05:49,625 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE 2024-11-18T19:05:49,625 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=CLOSING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:49,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE because future has completed 2024-11-18T19:05:49,628 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T19:05:49,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:49,780 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:49,781 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T19:05:49,781 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing b38dbac143e9407aa48a95e23085ace3, disabling compactions & flushes 2024-11-18T19:05:49,781 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:49,781 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:49,781 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. after waiting 0 ms 2024-11-18T19:05:49,781 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:49,787 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-18T19:05:49,787 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:05:49,788 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:49,788 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for b38dbac143e9407aa48a95e23085ace3: Waiting for close lock at 1731956749781Running coprocessor pre-close hooks at 1731956749781Disabling compacts and flushes for region at 1731956749781Disabling writes for close at 1731956749781Writing region close event to WAL at 1731956749782 (+1 ms)Running coprocessor post-close hooks at 1731956749787 (+5 ms)Closed at 1731956749787 2024-11-18T19:05:49,788 WARN [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: b38dbac143e9407aa48a95e23085ace3 to self. 2024-11-18T19:05:49,790 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:49,791 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=CLOSED 2024-11-18T19:05:49,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:49,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-18T19:05:49,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 in 166 msec 2024-11-18T19:05:49,796 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE; state=CLOSED, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=true 2024-11-18T19:05:49,947 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:49,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE because future has completed 2024-11-18T19:05:49,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538}] 2024-11-18T19:05:50,106 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:50,106 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => b38dbac143e9407aa48a95e23085ace3, NAME => 'test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:05:50,107 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:05:50,107 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731956706111 b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,107 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:05:50,107 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,107 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,109 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,110 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b38dbac143e9407aa48a95e23085ace3 columnFamilyName f 2024-11-18T19:05:50,110 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:50,117 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d 2024-11-18T19:05:50,122 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad 2024-11-18T19:05:50,122 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(327): Store=b38dbac143e9407aa48a95e23085ace3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:50,122 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,124 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b38dbac143e9407aa48a95e23085ace3 columnFamilyName f2 2024-11-18T19:05:50,124 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:50,125 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(327): Store=b38dbac143e9407aa48a95e23085ace3/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:50,125 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,126 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b38dbac143e9407aa48a95e23085ace3 columnFamilyName mob 2024-11-18T19:05:50,126 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:50,133 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 2024-11-18T19:05:50,137 DEBUG [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c 2024-11-18T19:05:50,137 INFO [StoreOpener-b38dbac143e9407aa48a95e23085ace3-1 {}] regionserver.HStore(327): Store=b38dbac143e9407aa48a95e23085ace3/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:05:50,138 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,138 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,139 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,140 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,140 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,141 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1731956706111 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T19:05:50,142 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,143 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened b38dbac143e9407aa48a95e23085ace3; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11382789280, jitterRate=0.0601048618555069}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T19:05:50,143 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:50,146 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for b38dbac143e9407aa48a95e23085ace3: Running coprocessor pre-open hook at 1731956750107Writing region info on filesystem at 1731956750107Initializing all the Stores at 1731956750108 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956750109 (+1 ms)Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956750109Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956750109Cleaning up temporary data from old regions at 1731956750140 (+31 ms)Running coprocessor post-open hooks at 1731956750143 (+3 ms)Region opened successfully at 1731956750146 (+3 ms) 2024-11-18T19:05:50,147 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., pid=46, masterSystemTime=1731956750101 2024-11-18T19:05:50,149 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:50,149 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:50,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=b38dbac143e9407aa48a95e23085ace3, regionState=OPEN, openSeqNum=338, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:05:50,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:05:50,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-18T19:05:50,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure b38dbac143e9407aa48a95e23085ace3, server=818bff808382,33643,1731956698538 in 203 msec 2024-11-18T19:05:50,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-18T19:05:50,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731956706111, region=b38dbac143e9407aa48a95e23085ace3, REOPEN/MOVE in 531 msec 2024-11-18T19:05:50,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-18T19:05:50,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1731956706111 in 535 msec 2024-11-18T19:05:50,160 DEBUG [PEWorker-5 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1731956706111 2024-11-18T19:05:50,167 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 2024-11-18T19:05:50,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1731956706111 in 967 msec 2024-11-18T19:05:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T19:05:50,171 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1731956706111 completed 2024-11-18T19:05:50,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,204 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-18T19:05:50,207 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731956750172 2024-11-18T19:05:50,209 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1731956736633} 2024-11-18T19:05:50,209 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731956714240} 2024-11-18T19:05:50,209 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-18T19:05:50,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,251 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1731956706111 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1731956706111 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T19:05:50,255 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1731956750172,startts=0,failedts=1731956750252,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1731956706111 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-18T19:05:50,255 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-18T19:05:50,257 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-18T19:05:50,258 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-18T19:05:50,259 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-18T19:05:50,259 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-18T19:05:50,261 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-18T19:05:50,263 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1731956750172 failed. 2024-11-18T19:05:50,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:50,281 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1731956706111, ns2:test-17319567061111, ns3:test-17319567061112, ns4:test-17319567061113] 2024-11-18T19:05:50,281 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-18T19:05:50,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-18T19:05:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956750286 (current time:1731956750286). 2024-11-18T19:05:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:05:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-18T19:05:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234e2ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:50,287 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:50,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:50,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:50,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53948ace, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:50,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:50,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:50,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:50,289 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:50,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c10d52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:50,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:50,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:50,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:50,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:05:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:50,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:05:50,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-18T19:05:50,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-18T19:05:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-18T19:05:50,296 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:05:50,297 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:05:50,300 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:05:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741905_1081 (size=52) 2024-11-18T19:05:50,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-18T19:05:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-18T19:05:50,706 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:05:50,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243}] 2024-11-18T19:05:50,708 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:50,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-11-18T19:05:50,860 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:50,861 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing 63e71206ef16a51a40e751c8e3988243 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-18T19:05:50,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4eb1dae1358249b3872bdade56e95504 is 158, key is trslm:hdfs://localhost:45363/backupUT\x00ns2:test-17319567061111/meta:log-roll-map/1731956748736/Put/seqid=0 2024-11-18T19:05:50,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741906_1082 (size=5745) 2024-11-18T19:05:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-18T19:05:51,271 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4eb1dae1358249b3872bdade56e95504 2024-11-18T19:05:51,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/2214a24780a548bb8ad4daf52f5d009c is 1245, key is session:backup_1731956750172/session:context/1731956750260/Put/seqid=0 2024-11-18T19:05:51,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741907_1083 (size=6682) 2024-11-18T19:05:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-18T19:05:51,686 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/2214a24780a548bb8ad4daf52f5d009c 2024-11-18T19:05:51,695 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4eb1dae1358249b3872bdade56e95504 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4eb1dae1358249b3872bdade56e95504 2024-11-18T19:05:51,701 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4eb1dae1358249b3872bdade56e95504, entries=4, sequenceid=31, filesize=5.6 K 2024-11-18T19:05:51,702 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/2214a24780a548bb8ad4daf52f5d009c as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/2214a24780a548bb8ad4daf52f5d009c 2024-11-18T19:05:51,710 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/2214a24780a548bb8ad4daf52f5d009c, entries=3, sequenceid=31, filesize=6.5 K 2024-11-18T19:05:51,712 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 63e71206ef16a51a40e751c8e3988243 in 851ms, sequenceid=31, compaction requested=true 2024-11-18T19:05:51,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for 63e71206ef16a51a40e751c8e3988243: 2024-11-18T19:05:51,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. for snapshot_backup_system completed. 2024-11-18T19:05:51,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.' region-info for snapshot=snapshot_backup_system 2024-11-18T19:05:51,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:05:51,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4519732d1ea9401aaac2675ef7be1464, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4eb1dae1358249b3872bdade56e95504] hfiles 2024-11-18T19:05:51,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4519732d1ea9401aaac2675ef7be1464 for snapshot=snapshot_backup_system 2024-11-18T19:05:51,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4eb1dae1358249b3872bdade56e95504 for snapshot=snapshot_backup_system 2024-11-18T19:05:51,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/82b014184a204d09b4c21c43a476139b, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/2214a24780a548bb8ad4daf52f5d009c] hfiles 2024-11-18T19:05:51,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889 for snapshot=snapshot_backup_system 2024-11-18T19:05:51,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/82b014184a204d09b4c21c43a476139b for snapshot=snapshot_backup_system 2024-11-18T19:05:51,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/2214a24780a548bb8ad4daf52f5d009c for snapshot=snapshot_backup_system 2024-11-18T19:05:51,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741908_1084 (size=251) 2024-11-18T19:05:51,985 DEBUG [master/818bff808382:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731956706111 2024-11-18T19:05:51,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741909_1085 (size=950) 2024-11-18T19:05:52,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:05:52,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-18T19:05:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-11-18T19:05:52,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:52,120 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:05:52,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-18T19:05:52,123 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:05:52,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 in 1.4150 sec 2024-11-18T19:05:52,124 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:05:52,125 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:05:52,125 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-18T19:05:52,126 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-18T19:05:52,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741910_1086 (size=888) 2024-11-18T19:05:52,398 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/.tabledesc/.tableinfo.0000000004.908 2024-11-18T19:05:52,398 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/.tabledesc/.tableinfo.0000000005.950 2024-11-18T19:05:52,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731956706111 in 415 msec 2024-11-18T19:05:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-18T19:05:52,540 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:05:52,549 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:05:52,549 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_backup_system 2024-11-18T19:05:52,551 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:05:52,551 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-18T19:05:52,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2570 sec 2024-11-18T19:05:54,297 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d144bc0c998ecf0ed9ce62cb631ee3eb, had cached 0 bytes from a total of 8413 2024-11-18T19:05:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-18T19:05:54,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-18T19:05:54,462 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731956750265 started at 1731956754462. 2024-11-18T19:05:54,464 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731956750265 has been started. 2024-11-18T19:05:54,468 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-18T19:05:54,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-18T19:05:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-18T19:05:54,470 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-18T19:05:54,470 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-18T19:05:54,470 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-18T19:05:54,470 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-18T19:05:54,471 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,471 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:54,473 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-18T19:05:54,473 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-18T19:05:54,473 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:05:54,473 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,473 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-18T19:05:54,474 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:54,474 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,475 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-18T19:05:54,475 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:54,475 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-18T19:05:54,476 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-18T19:05:54,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-18T19:05:54,476 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-18T19:05:54,476 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-18T19:05:54,476 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-18T19:05:54,476 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-18T19:05:54,476 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '818bff808382,33643,1731956698538' joining acquired barrier for procedure (rolllog) in zk 2024-11-18T19:05:54,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,478 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:54,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:54,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:54,478 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-18T19:05:54,478 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-18T19:05:54,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:54,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:54,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:54,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:54,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:54,481 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '818bff808382,33643,1731956698538' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-18T19:05:54,481 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2fabda80[Count = 0] remaining members to acquire global barrier 2024-11-18T19:05:54,481 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-18T19:05:54,481 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:54,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:54,482 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:54,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:54,482 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-18T19:05:54,483 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,483 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-18T19:05:54,483 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-18T19:05:54,483 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 818bff808382,33643,1731956698538 2024-11-18T19:05:54,483 INFO [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731956741010 highest: 1731956741010 on 818bff808382,33643,1731956698538 2024-11-18T19:05:54,483 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1:(num 1731956740961) roll requested 2024-11-18T19:05:54,497 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483, exclude list is [], retry=0 2024-11-18T19:05:54,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:54,502 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956740961 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 2024-11-18T19:05:54,502 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:54,502 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956740961 is not closed yet, will try archiving it next time 2024-11-18T19:05:54,502 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.meta:.meta(num 1731956740991) roll requested 2024-11-18T19:05:54,502 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956720999 2024-11-18T19:05:54,504 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956740961 not finished, retry = 0 2024-11-18T19:05:54,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741888_1064 (size=4927) 2024-11-18T19:05:54,516 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956754502.meta, exclude list is [], retry=0 2024-11-18T19:05:54,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:54,521 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956740991.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956754502.meta 2024-11-18T19:05:54,521 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:54,521 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956740991.meta is not closed yet, will try archiving it next time 2024-11-18T19:05:54,521 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0:(num 1731956741010) roll requested 2024-11-18T19:05:54,523 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956740991.meta not finished, retry = 0 2024-11-18T19:05:54,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741889_1065 (size=1903) 2024-11-18T19:05:54,536 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521, exclude list is [], retry=0 2024-11-18T19:05:54,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:05:54,541 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956741010 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 2024-11-18T19:05:54,541 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:05:54,541 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956741010 is not closed yet, will try archiving it next time 2024-11-18T19:05:54,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741890_1066 (size=93) 2024-11-18T19:05:54,543 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956741010 not finished, retry = 0 2024-11-18T19:05:54,543 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-18T19:05:54,543 INFO [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731956754521 on 818bff808382,33643,1731956698538 2024-11-18T19:05:54,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:05:54,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:05:54,560 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-18T19:05:54,560 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '818bff808382,33643,1731956698538' in zk 2024-11-18T19:05:54,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,562 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-18T19:05:54,562 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-18T19:05:54,562 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,562 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-18T19:05:54,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:54,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:54,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:54,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:54,564 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:54,565 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:54,565 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:54,566 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:54,566 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:54,567 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '818bff808382,33643,1731956698538': 2024-11-18T19:05:54,567 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-18T19:05:54,567 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '818bff808382,33643,1731956698538' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-18T19:05:54,567 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-18T19:05:54,567 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-18T19:05:54,567 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-18T19:05:54,567 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-18T19:05:54,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-18T19:05:54,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,570 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:05:54,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:05:54,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:05:54,571 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-18T19:05:54,571 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:05:54,571 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,571 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,571 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:54,572 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:05:54,572 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:54,573 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:54,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:05:54,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:05:54,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-18T19:05:54,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:05:54,576 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-18T19:05:54,576 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-18T19:05:54,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-18T19:05:54,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:05:54,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-18T19:05:54,577 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-11-18T19:05:54,577 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-18T19:05:54,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:05:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-18T19:05:54,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-18T19:05:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956754586 (current time:1731956754586). 2024-11-18T19:05:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:05:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731956754585_backup_system_bulk VERSION not specified, setting to 2 2024-11-18T19:05:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be1b397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:54,588 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:54,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:54,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:54,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60431de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:54,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:54,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:54,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:54,589 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:54,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59329a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:54,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:54,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:54,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:54,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:05:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:54,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:05:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-18T19:05:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-18T19:05:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T19:05:54,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:05:54,599 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:05:54,601 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:05:54,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741914_1090 (size=76) 2024-11-18T19:05:54,644 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956741010 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956741010 2024-11-18T19:05:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T19:05:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T19:05:55,009 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:05:55,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e689fbc273d3ab005a435f927e214efb}] 2024-11-18T19:05:55,010 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:55,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-18T19:05:55,162 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:55,162 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e689fbc273d3ab005a435f927e214efb: 2024-11-18T19:05:55,162 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. for snapshot_1731956754585_backup_system_bulk completed. 2024-11-18T19:05:55,163 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb.' region-info for snapshot=snapshot_1731956754585_backup_system_bulk 2024-11-18T19:05:55,163 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:05:55,163 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-18T19:05:55,163 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-18T19:05:55,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741915_1091 (size=61) 2024-11-18T19:05:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T19:05:55,330 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-18T19:05:55,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:05:55,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-18T19:05:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-18T19:05:55,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731956754585_backup_system_bulk on region e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:55,570 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e689fbc273d3ab005a435f927e214efb 2024-11-18T19:05:55,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-18T19:05:55,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure e689fbc273d3ab005a435f927e214efb in 562 msec 2024-11-18T19:05:55,573 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:05:55,574 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:05:55,575 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:05:55,575 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731956754585_backup_system_bulk 2024-11-18T19:05:55,576 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956754585_backup_system_bulk 2024-11-18T19:05:55,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741916_1092 (size=702) 2024-11-18T19:05:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T19:05:55,755 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T19:05:55,986 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:05:55,991 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:05:55,992 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956754585_backup_system_bulk to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956754585_backup_system_bulk 2024-11-18T19:05:55,993 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:05:55,993 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-18T19:05:55,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731956754585_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.3980 sec 2024-11-18T19:05:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T19:05:56,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-18T19:05:56,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } 2024-11-18T19:05:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956756744 (current time:1731956756744). 2024-11-18T19:05:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:05:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731956756743_default_test-1731956706111 VERSION not specified, setting to 2 2024-11-18T19:05:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69aa6124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:56,745 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:56,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:56,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:56,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ac119a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:56,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:56,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:56,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:56,747 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:56,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79750a00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:56,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:56,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:56,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:56,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:05:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:56,752 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:05:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } 2024-11-18T19:05:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-18T19:05:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T19:05:56,754 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:05:56,755 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:05:56,758 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:05:56,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741917_1093 (size=84) 2024-11-18T19:05:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T19:05:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T19:05:57,164 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:05:57,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b38dbac143e9407aa48a95e23085ace3}] 2024-11-18T19:05:57,165 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:57,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for b38dbac143e9407aa48a95e23085ace3: 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. for snapshot_1731956756743_default_test-1731956706111 completed. 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3.' region-info for snapshot=snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad] hfiles 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d for snapshot=snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad for snapshot=snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c] hfiles 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 for snapshot=snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:05:57,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c for snapshot=snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:05:57,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741918_1094 (size=224) 2024-11-18T19:05:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T19:05:57,725 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:05:57,725 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-18T19:05:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-18T19:05:57,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731956756743_default_test-1731956706111 on region b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:57,726 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:57,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-18T19:05:57,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure b38dbac143e9407aa48a95e23085ace3 in 563 msec 2024-11-18T19:05:57,728 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:05:57,729 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:05:57,730 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-18T19:05:57,731 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-18T19:05:57,731 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:57,731 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-18T19:05:57,731 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:57,732 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-18T19:05:57,732 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:05:57,733 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1/mob/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3] hfiles 2024-11-18T19:05:57,733 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1/mob/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:05:57,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741919_1095 (size=168) 2024-11-18T19:05:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T19:05:58,140 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:05:58,140 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:05:58,141 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:05:58,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741920_1096 (size=1347) 2024-11-18T19:05:58,556 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:05:58,565 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:05:58,566 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956756743_default_test-1731956706111 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:05:58,567 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:05:58,567 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-18T19:05:58,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731956756743_default_test-1731956706111 table=test-1731956706111 type=FLUSH ttl=0 } in 1.8150 sec 2024-11-18T19:05:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T19:05:58,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1731956706111 completed 2024-11-18T19:05:58,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } 2024-11-18T19:05:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956758903 (current time:1731956758903). 2024-11-18T19:05:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:05:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731956758903_ns2_test-17319567061111 VERSION not specified, setting to 2 2024-11-18T19:05:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23be3aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:05:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:05:58,905 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:05:58,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:05:58,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:05:58,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4852f95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:58,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:05:58,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:05:58,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:58,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:05:58,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c21724a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:05:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:05:58,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:05:58,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:05:58,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:05:58,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:05:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:05:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:05:58,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:05:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:05:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } 2024-11-18T19:05:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-18T19:05:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T19:05:58,915 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:05:58,916 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:05:58,919 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:05:58,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741921_1097 (size=86) 2024-11-18T19:05:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T19:05:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T19:05:59,326 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:05:59,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb}] 2024-11-18T19:05:59,327 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:05:59,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-18T19:05:59,479 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:05:59,479 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing d144bc0c998ecf0ed9ce62cb631ee3eb 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-18T19:05:59,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/.tmp/f/a5c01da247874f4a876ea5005d2e8711 is 38, key is row-t20/f:q1/1731956735270/Put/seqid=0 2024-11-18T19:05:59,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741922_1098 (size=5134) 2024-11-18T19:05:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T19:05:59,889 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/.tmp/f/a5c01da247874f4a876ea5005d2e8711 2024-11-18T19:05:59,896 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/.tmp/f/a5c01da247874f4a876ea5005d2e8711 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/a5c01da247874f4a876ea5005d2e8711 2024-11-18T19:05:59,902 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/a5c01da247874f4a876ea5005d2e8711, entries=5, sequenceid=111, filesize=5.0 K 2024-11-18T19:05:59,903 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for d144bc0c998ecf0ed9ce62cb631ee3eb in 423ms, sequenceid=111, compaction requested=false 2024-11-18T19:05:59,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for d144bc0c998ecf0ed9ce62cb631ee3eb: 2024-11-18T19:05:59,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. for snapshot_1731956758903_ns2_test-17319567061111 completed. 2024-11-18T19:05:59,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb.' region-info for snapshot=snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:05:59,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:05:59,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/a5c01da247874f4a876ea5005d2e8711] hfiles 2024-11-18T19:05:59,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 for snapshot=snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:05:59,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/a5c01da247874f4a876ea5005d2e8711 for snapshot=snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:05:59,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741923_1099 (size=130) 2024-11-18T19:06:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T19:06:00,309 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:06:00,309 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-18T19:06:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-18T19:06:00,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731956758903_ns2_test-17319567061111 on region d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:06:00,310 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:06:00,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-18T19:06:00,314 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:06:00,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d144bc0c998ecf0ed9ce62cb631ee3eb in 986 msec 2024-11-18T19:06:00,315 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:06:00,315 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:06:00,316 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:06:00,316 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:06:00,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741924_1100 (size=496) 2024-11-18T19:06:00,554 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 63e71206ef16a51a40e751c8e3988243, had cached 0 bytes from a total of 28925 2024-11-18T19:06:00,728 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:06:00,735 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:06:00,736 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956758903_ns2_test-17319567061111 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:06:00,737 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:06:00,737 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-18T19:06:00,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731956758903_ns2_test-17319567061111 table=ns2:test-17319567061111 type=FLUSH ttl=0 } in 1.8240 sec 2024-11-18T19:06:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-18T19:06:01,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17319567061111 completed 2024-11-18T19:06:01,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } 2024-11-18T19:06:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956761073 (current time:1731956761073). 2024-11-18T19:06:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:06:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731956761073_ns4_test-17319567061113 VERSION not specified, setting to 2 2024-11-18T19:06:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5f78b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:01,075 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:01,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:01,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:01,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ac8464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:01,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:01,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:01,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:01,077 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:01,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc84efa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:01,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:01,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:01,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:01,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:06:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:01,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:06:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } 2024-11-18T19:06:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-18T19:06:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T19:06:01,084 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:06:01,085 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:06:01,088 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:06:01,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741925_1101 (size=86) 2024-11-18T19:06:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T19:06:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T19:06:01,495 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:06:01,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f88044664c926e5a290147e827b7cf95}] 2024-11-18T19:06:01,496 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f88044664c926e5a290147e827b7cf95 2024-11-18T19:06:01,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-18T19:06:01,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:06:01,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for f88044664c926e5a290147e827b7cf95: 2024-11-18T19:06:01,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. for snapshot_1731956761073_ns4_test-17319567061113 completed. 2024-11-18T19:06:01,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95.' region-info for snapshot=snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:01,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:06:01,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-18T19:06:01,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741926_1102 (size=52) 2024-11-18T19:06:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T19:06:01,978 INFO [master/818bff808382:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T19:06:01,978 INFO [master/818bff808382:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T19:06:02,055 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:06:02,055 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-18T19:06:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-18T19:06:02,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731956761073_ns4_test-17319567061113 on region f88044664c926e5a290147e827b7cf95 2024-11-18T19:06:02,056 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f88044664c926e5a290147e827b7cf95 2024-11-18T19:06:02,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-18T19:06:02,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f88044664c926e5a290147e827b7cf95 in 562 msec 2024-11-18T19:06:02,059 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:06:02,060 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:06:02,060 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:06:02,061 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:02,061 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:02,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741927_1103 (size=443) 2024-11-18T19:06:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T19:06:02,472 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:06:02,479 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:06:02,479 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956761073_ns4_test-17319567061113 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:02,481 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:06:02,481 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-18T19:06:02,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731956761073_ns4_test-17319567061113 table=ns4:test-17319567061113 type=FLUSH ttl=0 } in 1.3990 sec 2024-11-18T19:06:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T19:06:03,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17319567061113 completed 2024-11-18T19:06:03,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } 2024-11-18T19:06:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956763233 (current time:1731956763233). 2024-11-18T19:06:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:06:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731956763233_ns3_test-17319567061112 VERSION not specified, setting to 2 2024-11-18T19:06:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6899eeca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:03,235 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:03,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:03,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:03,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fca0a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:03,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:03,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:03,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:03,237 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:03,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@466979be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:03,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:03,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:03,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:03,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:06:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:03,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:06:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } 2024-11-18T19:06:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-18T19:06:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T19:06:03,244 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:06:03,245 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:06:03,248 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:06:03,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741928_1104 (size=86) 2024-11-18T19:06:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T19:06:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T19:06:03,654 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:06:03,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f44dca6ed6ddff070a4eead3e92d66bc}] 2024-11-18T19:06:03,656 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:06:03,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-18T19:06:03,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:06:03,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f44dca6ed6ddff070a4eead3e92d66bc: 2024-11-18T19:06:03,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. for snapshot_1731956763233_ns3_test-17319567061112 completed. 2024-11-18T19:06:03,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc.' region-info for snapshot=snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:03,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:06:03,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-18T19:06:03,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741929_1105 (size=52) 2024-11-18T19:06:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T19:06:04,214 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:06:04,214 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-18T19:06:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-18T19:06:04,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731956763233_ns3_test-17319567061112 on region f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:06:04,215 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:06:04,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-18T19:06:04,218 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:06:04,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f44dca6ed6ddff070a4eead3e92d66bc in 561 msec 2024-11-18T19:06:04,219 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:06:04,220 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:06:04,220 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:04,221 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741930_1106 (size=443) 2024-11-18T19:06:04,232 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:06:04,238 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:06:04,239 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_1731956763233_ns3_test-17319567061112 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:04,240 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:06:04,240 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-18T19:06:04,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731956763233_ns3_test-17319567061112 table=ns3:test-17319567061112 type=FLUSH ttl=0 } in 999 msec 2024-11-18T19:06:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-18T19:06:04,382 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17319567061112 completed 2024-11-18T19:06:04,383 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1731956750265 2024-11-18T19:06:04,383 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-18T19:06:04,383 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-18T19:06:04,383 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731956750265_backup:system_bulk 2024-11-18T19:06:04,383 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731956754585_backup_system_bulk to hdfs://localhost:45363/backupUT/backup_1731956750265/backup/system_bulk/ 2024-11-18T19:06:04,383 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-18T19:06:04,432 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:45363, inputRoot=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:04,432 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1308346821_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45363/backupUT/backup_1731956750265/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45363/backupUT/backup_1731956750265/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1731956754585_backup_system_bulk 2024-11-18T19:06:04,456 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-18T19:06:04,473 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956754585_backup_system_bulk to hdfs://localhost:45363/backupUT/backup_1731956750265/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1731956754585_backup_system_bulk 2024-11-18T19:06:04,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741931_1107 (size=76) 2024-11-18T19:06:04,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741932_1108 (size=702) 2024-11-18T19:06:04,905 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:04,906 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:04,906 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,918 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-17510071230390764397.jar 2024-11-18T19:06:05,919 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,919 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,919 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,919 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,920 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,920 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,920 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,920 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,921 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:05,921 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:05,921 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:05,921 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:05,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:05,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:05,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:05,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:05,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:05,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:05,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:05,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:05,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:05,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:05,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:05,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:05,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:05,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:05,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:05,969 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:06,000 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731956754585_backup_system_bulk' hfile list 2024-11-18T19:06:06,096 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1937219540_0005_97bc3ea2-cbb1-48f2-9958-d9f3fd0f2cc0/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:06,096 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1937219540_0005_97bc3ea2-cbb1-48f2-9958-d9f3fd0f2cc0/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:07,138 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-18T19:06:07,139 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-18T19:06:07,144 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731956754585_backup_system_bulk 2024-11-18T19:06:07,144 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731956754585_backup_system_bulk finished. 2024-11-18T19:06:07,144 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731956750265_test-1731956706111 2024-11-18T19:06:07,144 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731956756743_default_test-1731956706111 to hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/ 2024-11-18T19:06:07,144 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-18T19:06:07,175 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:45363, inputRoot=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:07,176 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1308346821_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/.hbase-snapshot/.tmp/snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:06:07,178 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-18T19:06:07,186 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956756743_default_test-1731956706111 to hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/.hbase-snapshot/.tmp/snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:06:07,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741933_1109 (size=84) 2024-11-18T19:06:07,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741934_1110 (size=1347) 2024-11-18T19:06:07,612 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:07,612 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:07,613 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-11923116921545854346.jar 2024-11-18T19:06:08,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,617 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,617 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,617 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,617 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,618 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:08,618 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:08,618 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:08,618 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:08,619 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:08,619 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:08,619 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:08,619 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:08,620 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:08,620 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:08,620 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:08,620 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:08,621 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:08,621 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:08,621 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:08,621 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:08,622 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:08,622 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:08,622 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:08,662 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:08,688 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731956756743_default_test-1731956706111' hfile list 2024-11-18T19:06:08,691 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=43.0 K 2024-11-18T19:06:08,776 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local610072333_0006_e7cf4e7e-49e4-475b-8627-f2a516136503/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:08,776 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local610072333_0006_e7cf4e7e-49e4-475b-8627-f2a516136503/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:08,849 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-18T19:06:08,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741935_1111 (size=13760) 2024-11-18T19:06:09,294 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c output=hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c 2024-11-18T19:06:09,294 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-18T19:06:09,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741936_1112 (size=10796) 2024-11-18T19:06:09,728 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad output=hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad 2024-11-18T19:06:09,728 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-18T19:06:09,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741937_1113 (size=8413) 2024-11-18T19:06:10,170 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d output=hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d 2024-11-18T19:06:10,170 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-18T19:06:10,195 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1/mob/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3 to hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1/mob/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3" truncated to max limit (512 characters) 2024-11-18T19:06:10,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741938_1114 (size=5591) 2024-11-18T19:06:10,604 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1/mob/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3 output=hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1/mob/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:06:10,604 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5591 (5.5 K) time=0sec 0.012M/sec 2024-11-18T19:06:10,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741939_1115 (size=5484) 2024-11-18T19:06:11,045 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 output=hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 2024-11-18T19:06:11,045 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-18T19:06:11,819 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-18T19:06:11,820 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-18T19:06:11,828 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:06:11,828 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731956756743_default_test-1731956706111 finished. 2024-11-18T19:06:11,829 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731956750265_ns2:test-17319567061111 2024-11-18T19:06:11,829 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731956758903_ns2_test-17319567061111 to hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/ 2024-11-18T19:06:11,829 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-18T19:06:11,858 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:45363, inputRoot=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:11,859 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1308346821_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/.hbase-snapshot/.tmp/snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:06:11,860 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-18T19:06:11,865 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956758903_ns2_test-17319567061111 to hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/.hbase-snapshot/.tmp/snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:06:11,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741940_1116 (size=86) 2024-11-18T19:06:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741941_1117 (size=496) 2024-11-18T19:06:12,275 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:12,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:12,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,253 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-15392453253723399089.jar 2024-11-18T19:06:13,253 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,254 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,254 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,254 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,254 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,255 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,255 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,255 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,255 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:13,255 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:13,256 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:13,256 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:13,256 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:13,256 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:13,257 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:13,257 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:13,257 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:13,257 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:13,258 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:13,258 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:13,258 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:13,258 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:13,258 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:13,259 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:13,259 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:13,259 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:13,259 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:13,299 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:13,326 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731956758903_ns2_test-17319567061111' hfile list 2024-11-18T19:06:13,328 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=13.2 K 2024-11-18T19:06:13,415 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1780272400_0007_9363fb4d-c8d5-4a40-8a9f-67dbaca78099/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:13,415 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1780272400_0007_9363fb4d-c8d5-4a40-8a9f-67dbaca78099/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:13,485 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-18T19:06:13,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741942_1118 (size=8413) 2024-11-18T19:06:13,919 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 output=hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 2024-11-18T19:06:13,919 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-11-18T19:06:13,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741943_1119 (size=5134) 2024-11-18T19:06:14,360 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/a5c01da247874f4a876ea5005d2e8711 output=hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/a5c01da247874f4a876ea5005d2e8711 2024-11-18T19:06:14,360 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-11-18T19:06:14,455 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-18T19:06:14,456 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-18T19:06:14,462 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:06:14,462 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731956758903_ns2_test-17319567061111 finished. 2024-11-18T19:06:14,462 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731956750265_ns4:test-17319567061113 2024-11-18T19:06:14,462 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731956761073_ns4_test-17319567061113 to hdfs://localhost:45363/backupUT/backup_1731956750265/ns4/test-17319567061113/ 2024-11-18T19:06:14,462 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-18T19:06:14,491 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:45363, inputRoot=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:14,491 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1308346821_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45363/backupUT/backup_1731956750265/ns4/test-17319567061113, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45363/backupUT/backup_1731956750265/ns4/test-17319567061113/.hbase-snapshot/.tmp/snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:14,493 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-18T19:06:14,496 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956761073_ns4_test-17319567061113 to hdfs://localhost:45363/backupUT/backup_1731956750265/ns4/test-17319567061113/.hbase-snapshot/.tmp/snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:14,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741944_1120 (size=86) 2024-11-18T19:06:14,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741945_1121 (size=443) 2024-11-18T19:06:14,907 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:14,907 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:14,908 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,886 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-15809126626091374471.jar 2024-11-18T19:06:15,886 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,887 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,887 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,887 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,887 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,887 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,888 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,888 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,888 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:15,888 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:15,889 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:15,889 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:15,889 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:15,889 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:15,890 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:15,890 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:15,890 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:15,890 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:15,890 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:15,891 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:15,891 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:15,891 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:15,891 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:15,892 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:15,892 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:15,892 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:15,892 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:15,931 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:15,957 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731956761073_ns4_test-17319567061113' hfile list 2024-11-18T19:06:16,043 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local307657761_0008_b489bf31-f5fd-4779-8269-e1c95a9c77b9/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:16,044 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local307657761_0008_b489bf31-f5fd-4779-8269-e1c95a9c77b9/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:17,084 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-18T19:06:17,085 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-18T19:06:17,090 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:17,090 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731956761073_ns4_test-17319567061113 finished. 2024-11-18T19:06:17,090 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731956750265_ns3:test-17319567061112 2024-11-18T19:06:17,090 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731956763233_ns3_test-17319567061112 to hdfs://localhost:45363/backupUT/backup_1731956750265/ns3/test-17319567061112/ 2024-11-18T19:06:17,090 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-18T19:06:17,119 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:45363, inputRoot=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:17,119 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1308346821_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:45363/backupUT/backup_1731956750265/ns3/test-17319567061112, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:45363/backupUT/backup_1731956750265/ns3/test-17319567061112/.hbase-snapshot/.tmp/snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:17,121 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-18T19:06:17,125 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_1731956763233_ns3_test-17319567061112 to hdfs://localhost:45363/backupUT/backup_1731956750265/ns3/test-17319567061112/.hbase-snapshot/.tmp/snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:17,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741946_1122 (size=86) 2024-11-18T19:06:17,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741947_1123 (size=443) 2024-11-18T19:06:17,536 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:17,536 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:17,537 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,516 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-10757858019246263225.jar 2024-11-18T19:06:18,517 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,517 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,517 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,517 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,518 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,518 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,518 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,518 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,518 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:18,519 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:18,519 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:18,519 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:18,519 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:18,519 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:18,520 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:18,520 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:18,520 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:18,520 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:18,521 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:18,521 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:18,521 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:18,521 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:18,521 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:18,522 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:18,522 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-18T19:06:18,522 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:18,522 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:18,561 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:18,588 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731956763233_ns3_test-17319567061112' hfile list 2024-11-18T19:06:18,674 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local588632541_0009_e7d5dab7-4719-415f-8dc6-59157007050f/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:18,674 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local588632541_0009_e7d5dab7-4719-415f-8dc6-59157007050f/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:19,714 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-18T19:06:19,715 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-18T19:06:19,720 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:19,720 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731956763233_ns3_test-17319567061112 finished. 2024-11-18T19:06:19,720 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-18T19:06:19,720 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1731956706111 2024-11-18T19:06:19,720 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17319567061111 2024-11-18T19:06:19,720 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17319567061113 2024-11-18T19:06:19,720 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17319567061112 2024-11-18T19:06:19,730 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731956750265 2024-11-18T19:06:19,731 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-18T19:06:19,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741948_1124 (size=209) 2024-11-18T19:06:20,136 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45363/backupUT/backup_1731956750265/.backup.manifest 2024-11-18T19:06:20,136 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731956750265 finished: type=FULL,tablelist=backup:system_bulk;test-1731956706111;ns2:test-17319567061111;ns4:test-17319567061113;ns3:test-17319567061112,targetRootDir=hdfs://localhost:45363/backupUT,startts=1731956754462,completets=1731956779730,bytescopied=0 2024-11-18T19:06:20,136 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-18T19:06:20,136 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731956754585_backup_system_bulk 2024-11-18T19:06:20,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731956754585_backup_system_bulk" type: DISABLED 2024-11-18T19:06:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731956754585_backup_system_bulk 2024-11-18T19:06:20,141 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731956754585_backup_system_bulk for backup backup_1731956750265 succeeded. 2024-11-18T19:06:20,141 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:06:20,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731956756743_default_test-1731956706111" type: DISABLED 2024-11-18T19:06:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731956756743_default_test-1731956706111 2024-11-18T19:06:20,144 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731956756743_default_test-1731956706111 for backup backup_1731956750265 succeeded. 2024-11-18T19:06:20,144 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:06:20,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731956758903_ns2_test-17319567061111" type: DISABLED 2024-11-18T19:06:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731956758903_ns2_test-17319567061111 2024-11-18T19:06:20,147 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731956758903_ns2_test-17319567061111 for backup backup_1731956750265 succeeded. 2024-11-18T19:06:20,147 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:20,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731956761073_ns4_test-17319567061113" type: DISABLED 2024-11-18T19:06:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731956761073_ns4_test-17319567061113 2024-11-18T19:06:20,150 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731956761073_ns4_test-17319567061113 for backup backup_1731956750265 succeeded. 2024-11-18T19:06:20,150 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:20,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1731956763233_ns3_test-17319567061112" type: DISABLED 2024-11-18T19:06:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731956763233_ns3_test-17319567061112 2024-11-18T19:06:20,153 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731956763233_ns3_test-17319567061112 for backup backup_1731956750265 succeeded. 2024-11-18T19:06:20,153 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-18T19:06:20,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-18T19:06:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-18T19:06:20,159 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-18T19:06:20,161 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-18T19:06:20,167 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731956750265 completed. 2024-11-18T19:06:20,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,226 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-18T19:06:25,230 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731956785190 2024-11-18T19:06:25,233 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731956750265} 2024-11-18T19:06:25,233 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-18T19:06:25,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:25,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-18T19:06:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731956785320 (current time:1731956785320). 2024-11-18T19:06:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-18T19:06:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-18T19:06:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14fbc0c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:25,322 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:25,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:25,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:25,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3a44dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:25,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:25,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:25,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:25,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:25,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788a6791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:25,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:25,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:25,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:25,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649. 2024-11-18T19:06:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:25,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:25,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-18T19:06:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-18T19:06:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-18T19:06:25,331 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-18T19:06:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T19:06:25,332 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-18T19:06:25,334 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-18T19:06:25,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741949_1125 (size=52) 2024-11-18T19:06:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T19:06:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T19:06:25,740 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-18T19:06:25,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243}] 2024-11-18T19:06:25,742 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:06:25,755 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T19:06:25,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-18T19:06:25,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:06:25,894 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 63e71206ef16a51a40e751c8e3988243 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-18T19:06:25,899 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/f46926aba9b743f9b037bbef0ff1091b is 158, key is trslm:hdfs://localhost:45363/backupUT\x00ns2:test-17319567061111/meta:log-roll-map/1731956779725/Put/seqid=0 2024-11-18T19:06:25,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741950_1126 (size=6646) 2024-11-18T19:06:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T19:06:26,303 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/f46926aba9b743f9b037bbef0ff1091b 2024-11-18T19:06:26,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/7f60d8de1ddc43ee8a5c09f7943ab75b is 826, key is session:backup_1731956750265/session:context/1731956780160/Put/seqid=0 2024-11-18T19:06:26,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741951_1127 (size=5928) 2024-11-18T19:06:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T19:06:26,717 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/7f60d8de1ddc43ee8a5c09f7943ab75b 2024-11-18T19:06:26,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/f46926aba9b743f9b037bbef0ff1091b as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/f46926aba9b743f9b037bbef0ff1091b 2024-11-18T19:06:26,729 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/f46926aba9b743f9b037bbef0ff1091b, entries=12, sequenceid=42, filesize=6.5 K 2024-11-18T19:06:26,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/7f60d8de1ddc43ee8a5c09f7943ab75b as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/7f60d8de1ddc43ee8a5c09f7943ab75b 2024-11-18T19:06:26,735 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/7f60d8de1ddc43ee8a5c09f7943ab75b, entries=2, sequenceid=42, filesize=5.8 K 2024-11-18T19:06:26,736 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 63e71206ef16a51a40e751c8e3988243 in 841ms, sequenceid=42, compaction requested=true 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 63e71206ef16a51a40e751c8e3988243: 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. for snapshot_backup_system completed. 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243.' region-info for snapshot=snapshot_backup_system 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4519732d1ea9401aaac2675ef7be1464, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4eb1dae1358249b3872bdade56e95504, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/f46926aba9b743f9b037bbef0ff1091b] hfiles 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4519732d1ea9401aaac2675ef7be1464 for snapshot=snapshot_backup_system 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4eb1dae1358249b3872bdade56e95504 for snapshot=snapshot_backup_system 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/f46926aba9b743f9b037bbef0ff1091b for snapshot=snapshot_backup_system 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/82b014184a204d09b4c21c43a476139b, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/2214a24780a548bb8ad4daf52f5d009c, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/7f60d8de1ddc43ee8a5c09f7943ab75b] hfiles 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/86a0cd1ea2f44fae8d9f10d7f1c42889 for snapshot=snapshot_backup_system 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/82b014184a204d09b4c21c43a476139b for snapshot=snapshot_backup_system 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/2214a24780a548bb8ad4daf52f5d009c for snapshot=snapshot_backup_system 2024-11-18T19:06:26,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/7f60d8de1ddc43ee8a5c09f7943ab75b for snapshot=snapshot_backup_system 2024-11-18T19:06:26,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741952_1128 (size=330) 2024-11-18T19:06:27,142 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:06:27,142 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-18T19:06:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-18T19:06:27,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:06:27,143 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:06:27,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T19:06:27,146 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-18T19:06:27,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 63e71206ef16a51a40e751c8e3988243 in 1.4030 sec 2024-11-18T19:06:27,147 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-18T19:06:27,147 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-18T19:06:27,147 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-18T19:06:27,148 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-18T19:06:27,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741953_1129 (size=967) 2024-11-18T19:06:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T19:06:27,558 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-18T19:06:27,567 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-18T19:06:27,567 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.hbase-snapshot/snapshot_backup_system 2024-11-18T19:06:27,569 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-18T19:06:27,569 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-18T19:06:27,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2400 sec 2024-11-18T19:06:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T19:06:29,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-18T19:06:29,502 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731956785190 started at 1731956789502. 2024-11-18T19:06:29,504 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731956785190 has been started. 2024-11-18T19:06:29,510 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns4:test-17319567061113, backup:system_bulk, test-1731956706111, ns2:test-17319567061111, ns3:test-17319567061112] 2024-11-18T19:06:29,514 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1731956741010for backupID backup_1731956785190 2024-11-18T19:06:29,514 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-18T19:06:29,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-18T19:06:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-18T19:06:29,515 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-18T19:06:29,515 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-18T19:06:29,515 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-18T19:06:29,515 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-18T19:06:29,516 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,516 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-18T19:06:29,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-18T19:06:29,532 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,533 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-18T19:06:29,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:06:29,533 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,533 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-18T19:06:29,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-18T19:06:29,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,534 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-18T19:06:29,534 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-18T19:06:29,534 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-18T19:06:29,535 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-18T19:06:29,535 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-18T19:06:29,535 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-18T19:06:29,535 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-18T19:06:29,535 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-18T19:06:29,535 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-18T19:06:29,535 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '818bff808382,33643,1731956698538' joining acquired barrier for procedure (rolllog) in zk 2024-11-18T19:06:29,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,537 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-18T19:06:29,537 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,537 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:06:29,537 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:06:29,537 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-18T19:06:29,537 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-18T19:06:29,538 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:06:29,538 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:06:29,538 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:06:29,539 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:06:29,539 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:06:29,539 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '818bff808382,33643,1731956698538' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-18T19:06:29,539 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2a332340[Count = 0] remaining members to acquire global barrier 2024-11-18T19:06:29,539 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-18T19:06:29,539 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-18T19:06:29,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:06:29,541 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-18T19:06:29,541 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-18T19:06:29,541 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-18T19:06:29,541 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,541 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-18T19:06:29,541 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-18T19:06:29,542 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 818bff808382,33643,1731956698538 2024-11-18T19:06:29,542 INFO [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731956754521 highest: 1731956754521 on 818bff808382,33643,1731956698538 2024-11-18T19:06:29,542 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1:(num 1731956754483) roll requested 2024-11-18T19:06:29,555 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956789542, exclude list is [], retry=0 2024-11-18T19:06:29,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:06:29,560 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956789542 2024-11-18T19:06:29,560 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:06:29,560 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 is not closed yet, will try archiving it next time 2024-11-18T19:06:29,560 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.meta:.meta(num 1731956754502) roll requested 2024-11-18T19:06:29,560 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956740961 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956740961 2024-11-18T19:06:29,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741911_1087 (size=4710) 2024-11-18T19:06:29,562 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 not finished, retry = 0 2024-11-18T19:06:29,574 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956789560.meta, exclude list is [], retry=0 2024-11-18T19:06:29,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:06:29,578 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956754502.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956789560.meta 2024-11-18T19:06:29,578 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:06:29,578 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956754502.meta is not closed yet, will try archiving it next time 2024-11-18T19:06:29,578 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0:(num 1731956754521) roll requested 2024-11-18T19:06:29,579 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956754502.meta not finished, retry = 0 2024-11-18T19:06:29,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741912_1088 (size=93) 2024-11-18T19:06:29,591 DEBUG [regionserver/818bff808382:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956789578, exclude list is [], retry=0 2024-11-18T19:06:29,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33673,DS-0447fa01-1bd9-4334-a14d-506deb006e71,DISK] 2024-11-18T19:06:29,595 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956789578 2024-11-18T19:06:29,596 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-11-18T19:06:29,596 DEBUG [regionserver/818bff808382:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 is not closed yet, will try archiving it next time 2024-11-18T19:06:29,596 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956721049 2024-11-18T19:06:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741913_1089 (size=653) 2024-11-18T19:06:29,598 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 2024-11-18T19:06:29,602 DEBUG [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-18T19:06:29,602 INFO [rs(818bff808382,33643,1731956698538)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731956789578 on 818bff808382,33643,1731956698538 2024-11-18T19:06:29,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:29,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:29,620 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-18T19:06:29,620 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '818bff808382,33643,1731956698538' in zk 2024-11-18T19:06:29,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,622 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-18T19:06:29,622 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-18T19:06:29,622 DEBUG [member: '818bff808382,33643,1731956698538' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-18T19:06:29,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:06:29,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:06:29,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:06:29,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:06:29,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:06:29,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:06:29,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:06:29,625 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:06:29,625 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:06:29,626 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '818bff808382,33643,1731956698538': 2024-11-18T19:06:29,626 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-18T19:06:29,626 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '818bff808382,33643,1731956698538' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-18T19:06:29,626 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-18T19:06:29,626 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-18T19:06:29,626 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-18T19:06:29,626 INFO [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-18T19:06:29,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-18T19:06:29,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,628 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-18T19:06:29,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-18T19:06:29,629 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-18T19:06:29,629 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-18T19:06:29,629 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:06:29,629 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,629 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:06:29,629 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,629 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-18T19:06:29,630 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:06:29,630 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:06:29,630 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,630 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-18T19:06:29,631 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-18T19:06:29,631 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------818bff808382,33643,1731956698538 2024-11-18T19:06:29,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-18T19:06:29,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-18T19:06:29,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-18T19:06:29,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/818bff808382,33643,1731956698538 2024-11-18T19:06:29,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:06:29,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-18T19:06:29,633 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-18T19:06:29,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-18T19:06:29,633 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-18T19:06:29,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-18T19:06:29,634 DEBUG [(818bff808382,35649,1731956696665)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-18T19:06:29,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-18T19:06:29,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-18T19:06:29,634 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-18T19:06:29,634 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-18T19:06:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-18T19:06:29,645 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {818bff808382:33643=1731956741010} newestTimestamps: {818bff808382:33643=1731956754521} 2024-11-18T19:06:29,646 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 818bff808382%2C33643%2C1731956698538.meta.1731956701701.meta 2024-11-18T19:06:29,646 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 818bff808382%2C33643%2C1731956698538.meta.1731956740991.meta 2024-11-18T19:06:29,646 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 818bff808382%2C33643%2C1731956698538.meta.1731956754502.meta 2024-11-18T19:06:29,646 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 818bff808382%2C33643%2C1731956698538.meta.1731956789560.meta 2024-11-18T19:06:29,646 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956789578 2024-11-18T19:06:29,646 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 2024-11-18T19:06:29,646 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956789542 2024-11-18T19:06:29,647 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.meta.1731956721028.meta 2024-11-18T19:06:29,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741957_1133 (size=640) 2024-11-18T19:06:29,680 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956754502.meta to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.meta.1731956754502.meta 2024-11-18T19:06:30,058 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:45363/backupUT/backup_1731956785190/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:30,058 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-18T19:06:30,061 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-18T19:06:30,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741958_1134 (size=45) 2024-11-18T19:06:30,467 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-18T19:06:30,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741959_1135 (size=950) 2024-11-18T19:06:30,877 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1731956706111 target: hdfs://localhost:45363/backupUT/backup_1731956785190/default/test-1731956706111 descriptor: 'test-1731956706111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:30,878 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-18T19:06:30,880 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1731956706111 2024-11-18T19:06:30,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741960_1136 (size=53) 2024-11-18T19:06:31,285 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1731956706111 2024-11-18T19:06:31,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741961_1137 (size=364) 2024-11-18T19:06:31,697 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17319567061111 target: hdfs://localhost:45363/backupUT/backup_1731956785190/ns2/test-17319567061111 descriptor: 'ns2:test-17319567061111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:31,697 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-18T19:06:31,700 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17319567061111 2024-11-18T19:06:31,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741962_1138 (size=50) 2024-11-18T19:06:31,705 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17319567061111 2024-11-18T19:06:31,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741963_1139 (size=390) 2024-11-18T19:06:32,115 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17319567061113 target: hdfs://localhost:45363/backupUT/backup_1731956785190/ns4/test-17319567061113 descriptor: 'ns4:test-17319567061113', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:32,115 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-18T19:06:32,118 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17319567061113 2024-11-18T19:06:32,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741964_1140 (size=50) 2024-11-18T19:06:32,523 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17319567061113 2024-11-18T19:06:32,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741965_1141 (size=390) 2024-11-18T19:06:32,934 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17319567061112 target: hdfs://localhost:45363/backupUT/backup_1731956785190/ns3/test-17319567061112 descriptor: 'ns3:test-17319567061112', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:32,935 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-18T19:06:32,937 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17319567061112 2024-11-18T19:06:32,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741966_1142 (size=50) 2024-11-18T19:06:33,343 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17319567061112 2024-11-18T19:06:33,353 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:45363/backupUT/.tmp/backup_1731956785190 from hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483;hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 2024-11-18T19:06:33,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f97a6c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:33,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:33,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:33,355 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:33,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:33,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:33,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a663c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:33,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:33,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:33,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:33,357 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:33,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f18e948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:33,357 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:33,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:33,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7cf615e8 2024-11-18T19:06:33,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:33,359 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:33,362 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-18T19:06:33,362 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17319567061113 2024-11-18T19:06:33,362 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17319567061113,, stopping at row=ns4:test-17319567061113 ,, for max=2147483647 with caching=100 2024-11-18T19:06:33,363 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:33,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:33,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:33,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:33,366 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17319567061113: ns4:test-17319567061113; 2024-11-18T19:06:33,366 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-18T19:06:33,366 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:33,368 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-18T19:06:33,368 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1731956706111 2024-11-18T19:06:33,368 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731956706111,, stopping at row=test-1731956706111 ,, for max=2147483647 with caching=100 2024-11-18T19:06:33,370 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1731956706111: test-1731956706111; 2024-11-18T19:06:33,370 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17319567061111 2024-11-18T19:06:33,370 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17319567061111,, stopping at row=ns2:test-17319567061111 ,, for max=2147483647 with caching=100 2024-11-18T19:06:33,371 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17319567061111: ns2:test-17319567061111; 2024-11-18T19:06:33,371 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17319567061112 2024-11-18T19:06:33,371 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17319567061112,, stopping at row=ns3:test-17319567061112 ,, for max=2147483647 with caching=100 2024-11-18T19:06:33,372 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17319567061112: ns3:test-17319567061112; 2024-11-18T19:06:33,372 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-18T19:06:33,373 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/partitions_4baddffc-c361-4d09-8fdd-a9aa11c616a4 2024-11-18T19:06:33,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741967_1143 (size=324) 2024-11-18T19:06:33,780 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:33,780 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:33,780 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,763 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-13725008958360878293.jar 2024-11-18T19:06:34,764 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,764 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,764 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,764 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,765 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,765 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,765 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,765 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,765 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:34,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:34,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:34,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:34,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:34,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:34,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:34,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:34,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:34,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:34,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:34,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:34,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:34,769 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns4:test-17319567061113,backup:system_bulk,default:test-1731956706111,ns2:test-17319567061111,ns3:test-17319567061112 2024-11-18T19:06:34,769 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:34,770 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:34,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:34,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:34,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:34,770 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:35,107 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region b38dbac143e9407aa48a95e23085ace3, had cached 0 bytes from a total of 38453 2024-11-18T19:06:35,749 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-8476820759159893012.jar 2024-11-18T19:06:35,761 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:35,786 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 for WAL files 2024-11-18T19:06:35,787 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 2024-11-18T19:06:35,787 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 for WAL files 2024-11-18T19:06:35,788 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 2024-11-18T19:06:35,878 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1007237363_0010_22bb4ae0-eef0-4227-8d21-164661f21e83/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:35,879 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1007237363_0010_22bb4ae0-eef0-4227-8d21-164661f21e83/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:35,943 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 for hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956754483 (-9223372036854775808:9223372036854775807) length:4710 2024-11-18T19:06:35,947 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-18T19:06:35,947 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-18T19:06:35,993 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 for hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956754521 (-9223372036854775808:9223372036854775807) length:653 2024-11-18T19:06:35,996 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-18T19:06:35,996 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-18T19:06:36,089 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6821f01b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:36,089 DEBUG [pool-260-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:36,089 DEBUG [pool-260-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:36,090 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:36,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:36,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:36,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d0b011a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:36,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:36,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:36,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:36,092 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:36,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb38eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:36,093 DEBUG [pool-260-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:36,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:36,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:36,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:36,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731956706111', row='row-test-1731956706111-2-0', locateType=CURRENT is [region=test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., hostname=818bff808382,33643,1731956698538, seqNum=338] 2024-11-18T19:06:36,097 INFO [pool-260-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-260-thread-1. 2024-11-18T19:06:36,097 DEBUG [pool-260-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:36,097 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:36,098 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:36,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:36,098 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731956706111-2-0] 2024-11-18T19:06:36,098 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:36,102 DEBUG [pool-260-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/backupUT/.tmp/backup_1731956785190/_temporary/0/_temporary/attempt_local1007237363_0010_r_000004_0/default/test-1731956706111/f2/d6af71cf1d374f1d979dac1f181e260c is 58, key is row-test-1731956706111-2-0/f2:q1/1731956780182/Put/seqid=0 2024-11-18T19:06:36,104 WARN [Thread-1516 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:36,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741968_1144 (size=5367) 2024-11-18T19:06:36,923 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:45363/backupUT 2024-11-18T19:06:36,923 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1731956785190 2024-11-18T19:06:36,923 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-18T19:06:36,937 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:45363/backupUT/.tmp/backup_1731956785190, hdfs://localhost:45363/backupUT] 2024-11-18T19:06:36,984 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:37,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741969_1145 (size=5367) 2024-11-18T19:06:37,600 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-18T19:06:37,603 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1731956785190 2024-11-18T19:06:37,603 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 5367 bytes copied." 2024-11-18T19:06:37,603 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1059714079_0011 completed: true true 2024-11-18T19:06:37,604 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=6665902 FILE: Number of bytes written=17195105 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=410407 HDFS: Number of bytes written=2548285 HDFS: Number of read operations=1165 HDFS: Number of large read operations=0 HDFS: Number of write operations=421 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=1482 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=5367 Bytes Copied=5367 Bytes Expected=5367 Files Copied=2 DIR_COPY=4 2024-11-18T19:06:37,604 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:45363/backupUT/.tmp/backup_1731956785190 to hdfs://localhost:45363/backupUT finished. 2024-11-18T19:06:37,621 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731956785190 2024-11-18T19:06:37,624 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731956750265} 2024-11-18T19:06:37,625 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-18T19:06:37,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741970_1146 (size=714) 2024-11-18T19:06:38,030 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:45363/backupUT/backup_1731956785190/.backup.manifest 2024-11-18T19:06:38,030 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731956785190 finished: type=INCREMENTAL,tablelist=backup:system_bulk;test-1731956706111;ns2:test-17319567061111;ns4:test-17319567061113;ns3:test-17319567061112,targetRootDir=hdfs://localhost:45363/backupUT,startts=1731956789502,completets=1731956797621,bytescopied=0 2024-11-18T19:06:38,031 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-18T19:06:38,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-18T19:06:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-18T19:06:38,038 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-18T19:06:38,041 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-18T19:06:38,045 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731956785190 completed. 2024-11-18T19:06:38,045 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-18T19:06:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:38,060 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(191): Restoring full backup_1731956714240 2024-11-18T19:06:38,063 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table table1 does not exist. It will be created during restore process 2024-11-18T19:06:38,064 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table ns2:table2 does not exist. It will be created during restore process 2024-11-18T19:06:38,065 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45363/backupUT/backup_1731956714240/.backup.manifest 2024-11-18T19:06:38,067 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731956714240/.backup.manifest 2024-11-18T19:06:38,069 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45363/backupUT/backup_1731956714240/.backup.manifest 2024-11-18T19:06:38,071 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731956714240/.backup.manifest 2024-11-18T19:06:38,071 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1731956706111' to 'table1' from full backup image hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111 2024-11-18T19:06:38,079 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'table1' 2024-11-18T19:06:38,079 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:06:38,080 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f in region [hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3] 2024-11-18T19:06:38,084 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d first=row0 last=row98 2024-11-18T19:06:38,084 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3 in region [hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3] 2024-11-18T19:06:38,087 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 first=row-test-1731956706111-3-0 last=row-test-1731956706111-3-5 2024-11-18T19:06:38,087 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob in region [hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3] 2024-11-18T19:06:38,090 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 first=row-test-1731956706111-3-0 last=row-test-1731956706111-3-5 2024-11-18T19:06:38,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-18T19:06:38,098 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:06:38,098 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:38,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 62 2024-11-18T19:06:38,099 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:06:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T19:06:38,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741971_1147 (size=938) 2024-11-18T19:06:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T19:06:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T19:06:38,507 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a5f6dfde7b5aa4436340824f71ed826, NAME => 'table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:38,507 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3456776f3aeb3dfd204e897a25e061ce, NAME => 'table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:38,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741973_1149 (size=45) 2024-11-18T19:06:38,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741972_1148 (size=45) 2024-11-18T19:06:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing 3456776f3aeb3dfd204e897a25e061ce, disabling compactions & flushes 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 2a5f6dfde7b5aa4436340824f71ed826, disabling compactions & flushes 2024-11-18T19:06:38,914 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:38,914 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. after waiting 0 ms 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. after waiting 0 ms 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:38,914 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:38,914 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3456776f3aeb3dfd204e897a25e061ce: Waiting for close lock at 1731956798914Disabling compacts and flushes for region at 1731956798914Disabling writes for close at 1731956798914Writing region close event to WAL at 1731956798914Closed at 1731956798914 2024-11-18T19:06:38,914 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a5f6dfde7b5aa4436340824f71ed826: Waiting for close lock at 1731956798914Disabling compacts and flushes for region at 1731956798914Disabling writes for close at 1731956798914Writing region close event to WAL at 1731956798914Closed at 1731956798914 2024-11-18T19:06:38,915 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:06:38,916 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731956798915"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956798915"}]},"ts":"1731956798915"} 2024-11-18T19:06:38,916 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731956798915"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956798915"}]},"ts":"1731956798915"} 2024-11-18T19:06:38,919 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-11-18T19:06:38,920 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:06:38,920 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956798920"}]},"ts":"1731956798920"} 2024-11-18T19:06:38,922 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-18T19:06:38,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, ASSIGN}, {pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, ASSIGN}] 2024-11-18T19:06:38,924 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, ASSIGN 2024-11-18T19:06:38,924 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, ASSIGN 2024-11-18T19:06:38,925 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:06:38,925 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:06:39,075 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=2a5f6dfde7b5aa4436340824f71ed826, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:39,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=3456776f3aeb3dfd204e897a25e061ce, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:39,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, ASSIGN because future has completed 2024-11-18T19:06:39,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3456776f3aeb3dfd204e897a25e061ce, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:39,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, ASSIGN because future has completed 2024-11-18T19:06:39,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a5f6dfde7b5aa4436340824f71ed826, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T19:06:39,234 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:39,234 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 3456776f3aeb3dfd204e897a25e061ce, NAME => 'table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-18T19:06:39,234 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:06:39,234 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,235 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:39,235 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,235 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,236 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,238 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3456776f3aeb3dfd204e897a25e061ce columnFamilyName f 2024-11-18T19:06:39,238 DEBUG [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:39,238 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] regionserver.HStore(327): Store=3456776f3aeb3dfd204e897a25e061ce/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:39,238 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,240 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3456776f3aeb3dfd204e897a25e061ce columnFamilyName f3 2024-11-18T19:06:39,240 DEBUG [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:39,240 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] regionserver.HStore(327): Store=3456776f3aeb3dfd204e897a25e061ce/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:39,240 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,242 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3456776f3aeb3dfd204e897a25e061ce columnFamilyName mob 2024-11-18T19:06:39,243 DEBUG [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:39,243 INFO [StoreOpener-3456776f3aeb3dfd204e897a25e061ce-1 {}] regionserver.HStore(327): Store=3456776f3aeb3dfd204e897a25e061ce/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:39,243 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,244 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,244 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,245 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,245 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,246 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T19:06:39,247 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,249 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:06:39,250 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 3456776f3aeb3dfd204e897a25e061ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11479826720, jitterRate=0.0691421777009964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T19:06:39,250 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:39,250 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 3456776f3aeb3dfd204e897a25e061ce: Running coprocessor pre-open hook at 1731956799235Writing region info on filesystem at 1731956799235Initializing all the Stores at 1731956799236 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956799236Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956799236Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956799236Cleaning up temporary data from old regions at 1731956799245 (+9 ms)Running coprocessor post-open hooks at 1731956799250 (+5 ms)Region opened successfully at 1731956799250 2024-11-18T19:06:39,251 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce., pid=65, masterSystemTime=1731956799230 2024-11-18T19:06:39,254 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:39,254 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:39,254 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] handler.AssignRegionHandler(132): Open table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:39,254 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a5f6dfde7b5aa4436340824f71ed826, NAME => 'table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-18T19:06:39,254 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=3456776f3aeb3dfd204e897a25e061ce, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:39,254 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:06:39,254 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,255 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(898): Instantiated table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:39,255 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7794): checking encryption for 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,255 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7797): checking classloading for 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,256 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3456776f3aeb3dfd204e897a25e061ce, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:39,258 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a5f6dfde7b5aa4436340824f71ed826 columnFamilyName f 2024-11-18T19:06:39,258 DEBUG [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:39,258 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] regionserver.HStore(327): Store=2a5f6dfde7b5aa4436340824f71ed826/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:39,258 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-18T19:06:39,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 3456776f3aeb3dfd204e897a25e061ce, server=818bff808382,33643,1731956698538 in 179 msec 2024-11-18T19:06:39,260 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a5f6dfde7b5aa4436340824f71ed826 columnFamilyName f3 2024-11-18T19:06:39,260 DEBUG [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:39,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=62, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, ASSIGN in 337 msec 2024-11-18T19:06:39,261 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] regionserver.HStore(327): Store=2a5f6dfde7b5aa4436340824f71ed826/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:39,261 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,262 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a5f6dfde7b5aa4436340824f71ed826 columnFamilyName mob 2024-11-18T19:06:39,263 DEBUG [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:39,263 INFO [StoreOpener-2a5f6dfde7b5aa4436340824f71ed826-1 {}] regionserver.HStore(327): Store=2a5f6dfde7b5aa4436340824f71ed826/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:39,263 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1038): replaying wal for 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,264 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,264 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,265 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1048): stopping wal replay for 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,265 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1060): Cleaning up temporary data for 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,266 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T19:06:39,267 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1093): writing seq id for 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,269 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:06:39,270 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1114): Opened 2a5f6dfde7b5aa4436340824f71ed826; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10329022080, jitterRate=-0.03803485631942749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T19:06:39,270 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:39,270 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1006): Region open journal for 2a5f6dfde7b5aa4436340824f71ed826: Running coprocessor pre-open hook at 1731956799255Writing region info on filesystem at 1731956799255Initializing all the Stores at 1731956799256 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956799256Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956799256Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956799256Cleaning up temporary data from old regions at 1731956799265 (+9 ms)Running coprocessor post-open hooks at 1731956799270 (+5 ms)Region opened successfully at 1731956799270 2024-11-18T19:06:39,271 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., pid=66, masterSystemTime=1731956799230 2024-11-18T19:06:39,272 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:39,272 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] handler.AssignRegionHandler(153): Opened table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:39,273 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=2a5f6dfde7b5aa4436340824f71ed826, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:39,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a5f6dfde7b5aa4436340824f71ed826, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:39,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=63 2024-11-18T19:06:39,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a5f6dfde7b5aa4436340824f71ed826, server=818bff808382,33643,1731956698538 in 196 msec 2024-11-18T19:06:39,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-18T19:06:39,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, ASSIGN in 355 msec 2024-11-18T19:06:39,279 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:06:39,279 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956799279"}]},"ts":"1731956799279"} 2024-11-18T19:06:39,281 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-18T19:06:39,281 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:06:39,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 1.1860 sec 2024-11-18T19:06:39,297 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d144bc0c998ecf0ed9ce62cb631ee3eb, had cached 0 bytes from a total of 13547 2024-11-18T19:06:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T19:06:40,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-18T19:06:40,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:40,247 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 from hbase tables test-1731956706111 to tables table1 2024-11-18T19:06:40,248 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1731956706111 into table1 2024-11-18T19:06:40,250 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248 from hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:06:40,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@238ddc39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:40,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:40,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:40,252 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:40,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:40,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:40,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b43cab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:40,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:40,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:40,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:40,254 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:40,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6436504a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:40,255 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:40,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:40,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c58be49 2024-11-18T19:06:40,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:40,257 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:40,258 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-18T19:06:40,258 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-18T19:06:40,258 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:40,259 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:40,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:40,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:40,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:40,262 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-18T19:06:40,262 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-18T19:06:40,262 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-18T19:06:40,263 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/partitions_1a88b59a-da75-4527-9ec1-9c8fdd3529c2 2024-11-18T19:06:40,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741974_1150 (size=177) 2024-11-18T19:06:40,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:40,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:40,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-18207308651228077752.jar 2024-11-18T19:06:41,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:41,668 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:41,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:41,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:41,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:41,669 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:41,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:41,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:41,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:41,670 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:41,671 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:41,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:41,674 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:41,674 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-18T19:06:41,675 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:41,675 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:41,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:41,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:41,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:41,675 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-18T19:06:41,675 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:41,688 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:41,703 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-18T19:06:41,798 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1268999049_0012_5458bee9-b474-4adc-99fd-b91e27a3e4fe/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:41,798 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1268999049_0012_5458bee9-b474-4adc-99fd-b91e27a3e4fe/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:41,860 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d 2024-11-18T19:06:41,863 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:41,909 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 2024-11-18T19:06:41,913 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:41,946 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f3/b7ecceb2351a40c6895d36dc6ee8cc50 2024-11-18T19:06:41,949 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:41,980 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1270eaab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:41,980 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:41,980 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:41,981 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:41,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:41,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:41,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcdb587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:41,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:41,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:41,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:41,983 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:41,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bb8d30c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:41,984 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:41,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:41,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:41,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:41,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1731956706111-3-0', locateType=CURRENT is [region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:41,988 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-18T19:06:41,988 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:41,988 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:41,988 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:41,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:41,988 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731956706111-3-0] 2024-11-18T19:06:41,988 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:41,993 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565d546e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:41,993 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:41,993 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:41,994 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:41,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:41,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:41,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57361b72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:41,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:41,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:41,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:41,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:41,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4734ec9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:41,996 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:41,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:41,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:41,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:41,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1731956706111-3-0', locateType=CURRENT is [region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:41,999 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-18T19:06:41,999 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:41,999 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:41,999 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:41,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:41,999 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731956706111-3-0] 2024-11-18T19:06:41,999 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:42,003 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/_temporary/0/_temporary/attempt_local1268999049_0012_r_000000_0/f3/704553ada6194ab5b98a6388f64ea1cd is 58, key is row-test-1731956706111-3-0/f3:q1/1731956714211/Put/seqid=0 2024-11-18T19:06:42,004 WARN [Thread-1635 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:42,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741975_1151 (size=5530) 2024-11-18T19:06:42,008 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/_temporary/0/_temporary/attempt_local1268999049_0012_r_000000_0/mob/f9b21ed54e7644a18469ee02d6674730 is 59, key is row-test-1731956706111-3-0/mob:q1/1731956714226/Put/seqid=0 2024-11-18T19:06:42,010 WARN [Thread-1637 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:42,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741976_1152 (size=5539) 2024-11-18T19:06:42,438 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ae3711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:42,438 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:42,438 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:42,439 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:42,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:42,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:42,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4179dbbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:42,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:42,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:42,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:42,441 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:42,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b94485e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:42,442 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:42,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:42,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:42,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:42,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:42,445 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-18T19:06:42,445 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:42,445 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:42,445 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:42,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:42,445 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-18T19:06:42,446 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:42,451 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/_temporary/0/_temporary/attempt_local1268999049_0012_r_000001_0/f/4e5d095576e6447c9edb96eefb723a35 is 37, key is row10/f:q1/1731956708812/Put/seqid=0 2024-11-18T19:06:42,453 WARN [Thread-1652 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:42,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741977_1153 (size=8742) 2024-11-18T19:06:43,389 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-18T19:06:43,840 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248 2024-11-18T19:06:43,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d151ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:43,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:43,842 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:43,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:43,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:43,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c993751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:43,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:43,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,844 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:43,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302dd898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,844 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:43,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:43,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:43,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:43,850 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/_SUCCESS 2024-11-18T19:06:43,857 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:43,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:43,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44e9f1b4 2024-11-18T19:06:43,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:43,859 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:43,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:43,869 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,879 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/f/4e5d095576e6447c9edb96eefb723a35 first=Optional[row0] last=Optional[row98] 2024-11-18T19:06:43,882 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/f3/704553ada6194ab5b98a6388f64ea1cd first=Optional[row-test-1731956706111-3-0] last=Optional[row-test-1731956706111-3-5] 2024-11-18T19:06:43,885 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/mob/f9b21ed54e7644a18469ee02d6674730 first=Optional[row-test-1731956706111-3-0] last=Optional[row-test-1731956706111-3-5] 2024-11-18T19:06:43,903 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/f/4e5d095576e6447c9edb96eefb723a35 for inclusion in 3456776f3aeb3dfd204e897a25e061ce/f 2024-11-18T19:06:43,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/mob/f9b21ed54e7644a18469ee02d6674730 for inclusion in 2a5f6dfde7b5aa4436340824f71ed826/mob 2024-11-18T19:06:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row-test-1731956706111-3-0 last=row-test-1731956706111-3-5 2024-11-18T19:06:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-18T19:06:43,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/f3/704553ada6194ab5b98a6388f64ea1cd for inclusion in 2a5f6dfde7b5aa4436340824f71ed826/f3 2024-11-18T19:06:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-18T19:06:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-11-18T19:06:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HRegion(2603): Flush status journal for 3456776f3aeb3dfd204e897a25e061ce: 2024-11-18T19:06:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/f/4e5d095576e6447c9edb96eefb723a35 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f/4e5d095576e6447c9edb96eefb723a35 2024-11-18T19:06:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row-test-1731956706111-3-0 last=row-test-1731956706111-3-5 2024-11-18T19:06:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-18T19:06:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegion(2603): Flush status journal for 2a5f6dfde7b5aa4436340824f71ed826: 2024-11-18T19:06:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f/4e5d095576e6447c9edb96eefb723a35 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/f/72ad79c5fde34877a57ebd896b7b5983_SeqId_4_ 2024-11-18T19:06:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/mob/f9b21ed54e7644a18469ee02d6674730 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/mob/f9b21ed54e7644a18469ee02d6674730 2024-11-18T19:06:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5173b364, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:43,916 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:43,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:43,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:43,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e6f4a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:43,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:43,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/mob/f9b21ed54e7644a18469ee02d6674730 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/mob/2db7934a9792403cb51c83b45383f81c_SeqId_4_ 2024-11-18T19:06:43,917 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956800248/f3/704553ada6194ab5b98a6388f64ea1cd to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f3/704553ada6194ab5b98a6388f64ea1cd 2024-11-18T19:06:43,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22295ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:43,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:43,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@486885dd 2024-11-18T19:06:43,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:43,920 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f3/704553ada6194ab5b98a6388f64ea1cd as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/f3/6dc6fa6ea54e4a96a2ced4570b17c807_SeqId_4_ 2024-11-18T19:06:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f25d5a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:43,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:43,922 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:43,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:43,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:43,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:43,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd1ab2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:43,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:43,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:43,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:43,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752ef13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:43,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:43,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@623a9ece 2024-11-18T19:06:43,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:43,926 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:43,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:43,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:43,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:43,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:43,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:43,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643. 2024-11-18T19:06:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:43,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f/4e5d095576e6447c9edb96eefb723a35 into 3456776f3aeb3dfd204e897a25e061ce/f as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/f/72ad79c5fde34877a57ebd896b7b5983_SeqId_4_ - updating store file list. 2024-11-18T19:06:43,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f81ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:43,949 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:43,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:43,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:43,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6927925a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:43,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:43,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,950 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:43,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5b80c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:43,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:43,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3752f509 2024-11-18T19:06:43,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:43,952 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/f/72ad79c5fde34877a57ebd896b7b5983_SeqId_4_ into 3456776f3aeb3dfd204e897a25e061ce/f 2024-11-18T19:06:43,952 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f/4e5d095576e6447c9edb96eefb723a35 into 3456776f3aeb3dfd204e897a25e061ce/f (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/f/72ad79c5fde34877a57ebd896b7b5983_SeqId_4_) 2024-11-18T19:06:43,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f/4e5d095576e6447c9edb96eefb723a35 2024-11-18T19:06:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:43,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:43,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:43,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a9e2ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:43,959 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:43,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:43,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:43,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27fabfc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:43,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:43,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,962 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:43,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2713acfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:43,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:43,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7bf156ac 2024-11-18T19:06:43,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:43,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-18T19:06:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:43,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:43,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:43,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-18T19:06:43,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:43,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:43,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f3/704553ada6194ab5b98a6388f64ea1cd into 2a5f6dfde7b5aa4436340824f71ed826/f3 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/f3/6dc6fa6ea54e4a96a2ced4570b17c807_SeqId_4_ - updating store file list. 2024-11-18T19:06:43,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:43,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/f3/6dc6fa6ea54e4a96a2ced4570b17c807_SeqId_4_ into 2a5f6dfde7b5aa4436340824f71ed826/f3 2024-11-18T19:06:43,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f3/704553ada6194ab5b98a6388f64ea1cd into 2a5f6dfde7b5aa4436340824f71ed826/f3 (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/f3/6dc6fa6ea54e4a96a2ced4570b17c807_SeqId_4_) 2024-11-18T19:06:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/f3/704553ada6194ab5b98a6388f64ea1cd 2024-11-18T19:06:43,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/mob/f9b21ed54e7644a18469ee02d6674730 into 2a5f6dfde7b5aa4436340824f71ed826/mob as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/mob/2db7934a9792403cb51c83b45383f81c_SeqId_4_ - updating store file list. 2024-11-18T19:06:43,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:43,984 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643. 2024-11-18T19:06:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:43,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/mob/2db7934a9792403cb51c83b45383f81c_SeqId_4_ into 2a5f6dfde7b5aa4436340824f71ed826/mob 2024-11-18T19:06:43,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/mob/f9b21ed54e7644a18469ee02d6674730 into 2a5f6dfde7b5aa4436340824f71ed826/mob (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/mob/2db7934a9792403cb51c83b45383f81c_SeqId_4_) 2024-11-18T19:06:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__jjgu8rp4fbud3ds3023cd00mm9p76908v2o92s2knp093vcj6bt6bed9klvs13fp/mob/f9b21ed54e7644a18469ee02d6674730 2024-11-18T19:06:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a48639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:43,988 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:43,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:43,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:43,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453132be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:43,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:43,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:43,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:43,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e837bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:43,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:43,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a506a95 2024-11-18T19:06:43,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:43,992 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-18T19:06:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:43,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:43,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:43,994 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-18T19:06:43,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:44,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:44,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:44,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:44,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:44,012 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:44,012 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:44,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:44,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:44,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:44,013 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-18T19:06:44,013 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-18T19:06:44,013 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1731956714240 hdfs://localhost:45363/backupUT/backup_1731956714240/default/test-1731956706111/ 2024-11-18T19:06:44,014 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45363/backupUT/backup_1731956714240/.backup.manifest 2024-11-18T19:06:44,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2 , the old value is region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:06:44,014 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-18T19:06:44,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:06:44,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2 from cache 2024-11-18T19:06:44,016 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731956714240/.backup.manifest 2024-11-18T19:06:44,016 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17319567061111' to 'ns2:table2' from full backup image hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111 2024-11-18T19:06:44,024 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'ns2:table2' 2024-11-18T19:06:44,024 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:06:44,025 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f in region [hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb] 2024-11-18T19:06:44,028 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 first=row0 last=row98 2024-11-18T19:06:44,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:table2 2024-11-18T19:06:44,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T19:06:44,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 67 2024-11-18T19:06:44,032 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T19:06:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-18T19:06:44,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741978_1154 (size=351) 2024-11-18T19:06:44,122 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-18T19:06:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-18T19:06:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-18T19:06:44,439 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7cca13c6ec211bff31d3fc7f72705603, NAME => 'ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:44,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741979_1155 (size=37) 2024-11-18T19:06:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-18T19:06:44,758 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-18T19:06:44,846 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:44,846 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 7cca13c6ec211bff31d3fc7f72705603, disabling compactions & flushes 2024-11-18T19:06:44,846 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:44,846 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:44,846 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. after waiting 0 ms 2024-11-18T19:06:44,846 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:44,846 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:44,846 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7cca13c6ec211bff31d3fc7f72705603: Waiting for close lock at 1731956804846Disabling compacts and flushes for region at 1731956804846Disabling writes for close at 1731956804846Writing region close event to WAL at 1731956804846Closed at 1731956804846 2024-11-18T19:06:44,848 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T19:06:44,848 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1731956804848"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956804848"}]},"ts":"1731956804848"} 2024-11-18T19:06:44,850 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T19:06:44,850 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T19:06:44,851 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956804851"}]},"ts":"1731956804851"} 2024-11-18T19:06:44,852 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-11-18T19:06:44,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, ASSIGN}] 2024-11-18T19:06:44,853 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, ASSIGN 2024-11-18T19:06:44,854 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:06:45,005 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7cca13c6ec211bff31d3fc7f72705603, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:45,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, ASSIGN because future has completed 2024-11-18T19:06:45,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cca13c6ec211bff31d3fc7f72705603, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:45,163 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:45,163 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cca13c6ec211bff31d3fc7f72705603, NAME => 'ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:06:45,164 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:06:45,164 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,164 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:45,164 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,164 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,166 INFO [StoreOpener-7cca13c6ec211bff31d3fc7f72705603-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,167 INFO [StoreOpener-7cca13c6ec211bff31d3fc7f72705603-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cca13c6ec211bff31d3fc7f72705603 columnFamilyName f 2024-11-18T19:06:45,167 DEBUG [StoreOpener-7cca13c6ec211bff31d3fc7f72705603-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:45,168 INFO [StoreOpener-7cca13c6ec211bff31d3fc7f72705603-1 {}] regionserver.HStore(327): Store=7cca13c6ec211bff31d3fc7f72705603/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:45,168 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,169 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,169 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,169 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,169 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,171 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-18T19:06:45,173 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:06:45,173 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 7cca13c6ec211bff31d3fc7f72705603; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10499087520, jitterRate=-0.022196277976036072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:06:45,173 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:45,174 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 7cca13c6ec211bff31d3fc7f72705603: Running coprocessor pre-open hook at 1731956805164Writing region info on filesystem at 1731956805164Initializing all the Stores at 1731956805165 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956805165Cleaning up temporary data from old regions at 1731956805169 (+4 ms)Running coprocessor post-open hooks at 1731956805173 (+4 ms)Region opened successfully at 1731956805174 (+1 ms) 2024-11-18T19:06:45,175 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., pid=69, masterSystemTime=1731956805160 2024-11-18T19:06:45,177 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:45,177 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:45,177 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7cca13c6ec211bff31d3fc7f72705603, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:45,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cca13c6ec211bff31d3fc7f72705603, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:45,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-18T19:06:45,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7cca13c6ec211bff31d3fc7f72705603, server=818bff808382,33643,1731956698538 in 171 msec 2024-11-18T19:06:45,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-18T19:06:45,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, ASSIGN in 329 msec 2024-11-18T19:06:45,183 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T19:06:45,183 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956805183"}]},"ts":"1731956805183"} 2024-11-18T19:06:45,185 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-11-18T19:06:45,185 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T19:06:45,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:table2 in 1.1560 sec 2024-11-18T19:06:45,554 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 63e71206ef16a51a40e751c8e3988243, had cached 0 bytes from a total of 41499 2024-11-18T19:06:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-18T19:06:46,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:table2 completed 2024-11-18T19:06:46,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-18T19:06:46,186 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb from hbase tables ns2:test-17319567061111 to tables ns2:table2 2024-11-18T19:06:46,186 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17319567061111 into ns2:table2 2024-11-18T19:06:46,188 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956806186 from hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:06:46,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@236b208c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:46,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:46,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:46,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:46,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:46,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:46,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683fb2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:46,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:46,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:46,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:46,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:46,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f351c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:46,192 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:46,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:46,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e8b9ec3 2024-11-18T19:06:46,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:46,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:46,194 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-18T19:06:46,194 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-11-18T19:06:46,195 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-18T19:06:46,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:46,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:46,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:46,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:46,198 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-11-18T19:06:46,198 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-11-18T19:06:46,198 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/partitions_fed85790-95b1-4dcd-8270-d9419873c809 2024-11-18T19:06:46,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741980_1156 (size=153) 2024-11-18T19:06:46,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:46,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:46,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-11-18T19:06:47,103 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2024-11-18T19:06:47,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-11-18T19:06:47,104 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2024-11-18T19:06:47,585 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-16277267400784127139.jar 2024-11-18T19:06:47,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:47,588 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:47,588 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:47,588 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:47,588 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:47,588 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:47,588 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:47,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:47,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:47,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:47,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:47,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:47,591 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:47,591 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-11-18T19:06:47,591 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:47,591 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:47,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:47,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:47,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:47,591 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-18T19:06:47,591 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:47,604 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:47,616 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-18T19:06:47,712 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local89220726_0013_18fe2c8a-ba4b-4703-aa7c-ef5b1c0f25f8/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:47,712 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local89220726_0013_18fe2c8a-ba4b-4703-aa7c-ef5b1c0f25f8/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:47,778 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 2024-11-18T19:06:47,782 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:47,814 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd8fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:47,814 DEBUG [pool-272-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:47,814 DEBUG [pool-272-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:47,815 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:47,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:47,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:47,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5eeb89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:47,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:47,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:47,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:47,817 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:47,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c99aa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:47,818 DEBUG [pool-272-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:47,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:47,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:47,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:47,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row0', locateType=CURRENT is [region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:47,823 INFO [pool-272-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-272-thread-1. 2024-11-18T19:06:47,823 DEBUG [pool-272-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:47,823 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:47,823 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:47,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:47,823 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-18T19:06:47,823 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:47,828 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956806186/_temporary/0/_temporary/attempt_local89220726_0013_r_000000_0/f/d975bc490ad24bf5a101349d524ce4fa is 37, key is row10/f:q1/1731956709595/Put/seqid=0 2024-11-18T19:06:47,830 WARN [Thread-1719 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:47,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741981_1157 (size=8740) 2024-11-18T19:06:48,753 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956806186 2024-11-18T19:06:48,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5908c2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:48,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:48,755 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:48,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:48,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:48,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efc0251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:48,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:48,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:48,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a35296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,757 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:48,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:48,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:48,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:48,762 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956806186/_SUCCESS 2024-11-18T19:06:48,764 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:48,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:48,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c455d86 2024-11-18T19:06:48,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:48,766 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:48,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:48,771 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-18T19:06:48,776 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956806186/f/d975bc490ad24bf5a101349d524ce4fa first=Optional[row0] last=Optional[row98] 2024-11-18T19:06:48,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956806186/f/d975bc490ad24bf5a101349d524ce4fa for inclusion in 7cca13c6ec211bff31d3fc7f72705603/f 2024-11-18T19:06:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-18T19:06:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-18T19:06:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegion(2603): Flush status journal for 7cca13c6ec211bff31d3fc7f72705603: 2024-11-18T19:06:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956806186/f/d975bc490ad24bf5a101349d524ce4fa to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__sh9dvpk7rch101cnm081butappinogbl3jora6ue85plt9ha7pcbu8hknblmnb96/f/d975bc490ad24bf5a101349d524ce4fa 2024-11-18T19:06:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__sh9dvpk7rch101cnm081butappinogbl3jora6ue85plt9ha7pcbu8hknblmnb96/f/d975bc490ad24bf5a101349d524ce4fa as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/f/7575ecb710b64383be65dd5eb14e741f_SeqId_4_ 2024-11-18T19:06:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@836a752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:48,787 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:48,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:48,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:48,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9ff905, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:48,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:48,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,789 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:48,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d15281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:48,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:48,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4097ddb7 2024-11-18T19:06:48,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:48,791 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:48,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:48,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:48,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:48,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:48,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:48,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:48,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:48,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__sh9dvpk7rch101cnm081butappinogbl3jora6ue85plt9ha7pcbu8hknblmnb96/f/d975bc490ad24bf5a101349d524ce4fa into 7cca13c6ec211bff31d3fc7f72705603/f as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/f/7575ecb710b64383be65dd5eb14e741f_SeqId_4_ - updating store file list. 2024-11-18T19:06:48,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/f/7575ecb710b64383be65dd5eb14e741f_SeqId_4_ into 7cca13c6ec211bff31d3fc7f72705603/f 2024-11-18T19:06:48,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__sh9dvpk7rch101cnm081butappinogbl3jora6ue85plt9ha7pcbu8hknblmnb96/f/d975bc490ad24bf5a101349d524ce4fa into 7cca13c6ec211bff31d3fc7f72705603/f (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/f/7575ecb710b64383be65dd5eb14e741f_SeqId_4_) 2024-11-18T19:06:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__sh9dvpk7rch101cnm081butappinogbl3jora6ue85plt9ha7pcbu8hknblmnb96/f/d975bc490ad24bf5a101349d524ce4fa 2024-11-18T19:06:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3009b908, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:48,817 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:48,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:48,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:48,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5a64b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:48,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:48,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,818 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:48,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5b71a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:48,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:48,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6499367d 2024-11-18T19:06:48,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:48,820 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-18T19:06:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:48,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:48,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:48,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-18T19:06:48,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:48,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:48,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:48,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:48,837 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:48,837 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:48,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:48,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:48,837 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-18T19:06:48,838 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-18T19:06:48,838 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1731956714240 hdfs://localhost:45363/backupUT/backup_1731956714240/ns2/test-17319567061111/ 2024-11-18T19:06:48,838 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-11-18T19:06:48,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2 , the old value is region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:06:48,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:06:48,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2 from cache 2024-11-18T19:06:48,838 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-18T19:06:48,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:48,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:48,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:48,859 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45363/backupUT/backup_1731956785190/.backup.manifest 2024-11-18T19:06:48,861 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731956785190/.backup.manifest 2024-11-18T19:06:48,862 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45363/backupUT/backup_1731956750265/.backup.manifest 2024-11-18T19:06:48,863 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731956750265/.backup.manifest 2024-11-18T19:06:48,863 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1731956706111' to 'table1' from full backup image hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111 2024-11-18T19:06:48,865 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'test-1731956706111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1731956785190 2024-11-18T19:06:48,868 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'table1', preserving region splits 2024-11-18T19:06:48,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable table1 2024-11-18T19:06:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=table1 2024-11-18T19:06:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T19:06:48,881 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956808880"}]},"ts":"1731956808880"} 2024-11-18T19:06:48,882 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLING in hbase:meta 2024-11-18T19:06:48,882 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set table1 to state=DISABLING 2024-11-18T19:06:48,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=table1}] 2024-11-18T19:06:48,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, UNASSIGN}, {pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, UNASSIGN}] 2024-11-18T19:06:48,890 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, UNASSIGN 2024-11-18T19:06:48,890 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, UNASSIGN 2024-11-18T19:06:48,891 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=3456776f3aeb3dfd204e897a25e061ce, regionState=CLOSING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:48,891 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=2a5f6dfde7b5aa4436340824f71ed826, regionState=CLOSING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:48,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, UNASSIGN because future has completed 2024-11-18T19:06:48,893 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T19:06:48,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3456776f3aeb3dfd204e897a25e061ce, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:48,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, UNASSIGN because future has completed 2024-11-18T19:06:48,894 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T19:06:48,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a5f6dfde7b5aa4436340824f71ed826, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:48,942 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-18T19:06:48,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T19:06:49,045 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:49,045 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T19:06:49,045 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing 2a5f6dfde7b5aa4436340824f71ed826, disabling compactions & flushes 2024-11-18T19:06:49,045 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:49,045 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:49,045 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. after waiting 0 ms 2024-11-18T19:06:49,045 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:49,050 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-18T19:06:49,050 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:06:49,050 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. 2024-11-18T19:06:49,051 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for 2a5f6dfde7b5aa4436340824f71ed826: Waiting for close lock at 1731956809045Running coprocessor pre-close hooks at 1731956809045Disabling compacts and flushes for region at 1731956809045Disabling writes for close at 1731956809045Writing region close event to WAL at 1731956809046 (+1 ms)Running coprocessor post-close hooks at 1731956809050 (+4 ms)Closed at 1731956809050 2024-11-18T19:06:49,052 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed 2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:49,052 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(122): Close 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:49,052 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T19:06:49,052 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1722): Closing 3456776f3aeb3dfd204e897a25e061ce, disabling compactions & flushes 2024-11-18T19:06:49,052 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1755): Closing region table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:49,052 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:49,053 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. after waiting 1 ms 2024-11-18T19:06:49,053 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:49,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=2a5f6dfde7b5aa4436340824f71ed826, regionState=CLOSED 2024-11-18T19:06:49,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a5f6dfde7b5aa4436340824f71ed826, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:49,056 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-18T19:06:49,057 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:06:49,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=72 2024-11-18T19:06:49,057 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1973): Closed table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. 2024-11-18T19:06:49,057 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1676): Region close journal for 3456776f3aeb3dfd204e897a25e061ce: Waiting for close lock at 1731956809052Running coprocessor pre-close hooks at 1731956809052Disabling compacts and flushes for region at 1731956809052Disabling writes for close at 1731956809053 (+1 ms)Writing region close event to WAL at 1731956809053Running coprocessor post-close hooks at 1731956809057 (+4 ms)Closed at 1731956809057 2024-11-18T19:06:49,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=72, state=SUCCESS, hasLock=false; CloseRegionProcedure 2a5f6dfde7b5aa4436340824f71ed826, server=818bff808382,33643,1731956698538 in 161 msec 2024-11-18T19:06:49,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=2a5f6dfde7b5aa4436340824f71ed826, UNASSIGN in 168 msec 2024-11-18T19:06:49,059 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(157): Closed 3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:49,059 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=3456776f3aeb3dfd204e897a25e061ce, regionState=CLOSED 2024-11-18T19:06:49,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3456776f3aeb3dfd204e897a25e061ce, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:49,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-18T19:06:49,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; CloseRegionProcedure 3456776f3aeb3dfd204e897a25e061ce, server=818bff808382,33643,1731956698538 in 168 msec 2024-11-18T19:06:49,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=71 2024-11-18T19:06:49,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=3456776f3aeb3dfd204e897a25e061ce, UNASSIGN in 174 msec 2024-11-18T19:06:49,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-18T19:06:49,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=table1 in 181 msec 2024-11-18T19:06:49,067 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956809067"}]},"ts":"1731956809067"} 2024-11-18T19:06:49,069 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLED in hbase:meta 2024-11-18T19:06:49,069 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set table1 to state=DISABLED 2024-11-18T19:06:49,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DisableTableProcedure table=table1 in 195 msec 2024-11-18T19:06:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T19:06:49,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:table1 completed 2024-11-18T19:06:49,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.2 truncate table1 2024-11-18T19:06:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) 2024-11-18T19:06:49,213 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'table1' regions in transition 2024-11-18T19:06:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T19:06:49,218 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:49,219 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:49,222 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/f, FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/f3, FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/mob, FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/recovered.edits] 2024-11-18T19:06:49,222 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/f, FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/f3, FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/mob, FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/recovered.edits] 2024-11-18T19:06:49,226 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/f/72ad79c5fde34877a57ebd896b7b5983_SeqId_4_ to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/archive/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/f/72ad79c5fde34877a57ebd896b7b5983_SeqId_4_ 2024-11-18T19:06:49,227 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/f3/6dc6fa6ea54e4a96a2ced4570b17c807_SeqId_4_ to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/archive/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/f3/6dc6fa6ea54e4a96a2ced4570b17c807_SeqId_4_ 2024-11-18T19:06:49,230 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/mob/2db7934a9792403cb51c83b45383f81c_SeqId_4_ to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/archive/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/mob/2db7934a9792403cb51c83b45383f81c_SeqId_4_ 2024-11-18T19:06:49,231 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/recovered.edits/6.seqid to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/archive/data/default/table1/3456776f3aeb3dfd204e897a25e061ce/recovered.edits/6.seqid 2024-11-18T19:06:49,231 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/3456776f3aeb3dfd204e897a25e061ce 2024-11-18T19:06:49,232 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/recovered.edits/6.seqid to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/archive/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826/recovered.edits/6.seqid 2024-11-18T19:06:49,233 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/2a5f6dfde7b5aa4436340824f71ed826 2024-11-18T19:06:49,233 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived table1 regions 2024-11-18T19:06:49,234 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-11-18T19:06:49,234 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f/mob] 2024-11-18T19:06:49,236 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-11-18T19:06:49,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33643 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-18T19:06:49,247 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 2 rows of table1 from hbase:meta 2024-11-18T19:06:49,250 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'table1' descriptor. 2024-11-18T19:06:49,250 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'table1' from region states. 2024-11-18T19:06:49,251 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731956809251"}]},"ts":"9223372036854775807"} 2024-11-18T19:06:49,251 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731956809251"}]},"ts":"9223372036854775807"} 2024-11-18T19:06:49,253 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 2 regions from META 2024-11-18T19:06:49,253 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2a5f6dfde7b5aa4436340824f71ed826, NAME => 'table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826.', STARTKEY => '', ENDKEY => 'row0'}, {ENCODED => 3456776f3aeb3dfd204e897a25e061ce, NAME => 'table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce.', STARTKEY => 'row0', ENDKEY => ''}] 2024-11-18T19:06:49,253 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'table1' as deleted. 2024-11-18T19:06:49,253 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731956809253"}]},"ts":"9223372036854775807"} 2024-11-18T19:06:49,254 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table table1 state from META 2024-11-18T19:06:49,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741982_1158 (size=938) 2024-11-18T19:06:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T19:06:49,391 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2024-11-18T19:06:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T19:06:49,663 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf5451ece9935ba018e98673baf4e95b, NAME => 'table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:49,663 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => cd0db3f2ace15f2ccb5d44bb7a079f9c, NAME => 'table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:49,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741983_1159 (size=45) 2024-11-18T19:06:49,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741984_1160 (size=45) 2024-11-18T19:06:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T19:06:50,070 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:50,070 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing cf5451ece9935ba018e98673baf4e95b, disabling compactions & flushes 2024-11-18T19:06:50,070 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:50,070 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:50,070 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. after waiting 0 ms 2024-11-18T19:06:50,070 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:50,070 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:50,070 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf5451ece9935ba018e98673baf4e95b: Waiting for close lock at 1731956810070Disabling compacts and flushes for region at 1731956810070Disabling writes for close at 1731956810070Writing region close event to WAL at 1731956810070Closed at 1731956810070 2024-11-18T19:06:50,074 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:50,074 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing cd0db3f2ace15f2ccb5d44bb7a079f9c, disabling compactions & flushes 2024-11-18T19:06:50,074 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:50,074 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:50,074 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. after waiting 0 ms 2024-11-18T19:06:50,074 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:50,074 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:50,074 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for cd0db3f2ace15f2ccb5d44bb7a079f9c: Waiting for close lock at 1731956810074Disabling compacts and flushes for region at 1731956810074Disabling writes for close at 1731956810074Writing region close event to WAL at 1731956810074Closed at 1731956810074 2024-11-18T19:06:50,076 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731956810075"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956810075"}]},"ts":"1731956810075"} 2024-11-18T19:06:50,076 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731956810075"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956810075"}]},"ts":"1731956810075"} 2024-11-18T19:06:50,078 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-11-18T19:06:50,079 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956810079"}]},"ts":"1731956810079"} 2024-11-18T19:06:50,080 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-18T19:06:50,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, ASSIGN}, {pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, ASSIGN}] 2024-11-18T19:06:50,082 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, ASSIGN 2024-11-18T19:06:50,082 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, ASSIGN 2024-11-18T19:06:50,082 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:06:50,083 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:06:50,233 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=cd0db3f2ace15f2ccb5d44bb7a079f9c, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:50,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=cf5451ece9935ba018e98673baf4e95b, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:50,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, ASSIGN because future has completed 2024-11-18T19:06:50,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:50,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, ASSIGN because future has completed 2024-11-18T19:06:50,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T19:06:50,390 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:50,390 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => cf5451ece9935ba018e98673baf4e95b, NAME => 'table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-18T19:06:50,390 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:06:50,390 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,390 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:50,391 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,391 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,392 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,393 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5451ece9935ba018e98673baf4e95b columnFamilyName f 2024-11-18T19:06:50,393 DEBUG [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:50,394 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(327): Store=cf5451ece9935ba018e98673baf4e95b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:50,394 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,395 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5451ece9935ba018e98673baf4e95b columnFamilyName f3 2024-11-18T19:06:50,395 DEBUG [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:50,395 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(327): Store=cf5451ece9935ba018e98673baf4e95b/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:50,395 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,396 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5451ece9935ba018e98673baf4e95b columnFamilyName mob 2024-11-18T19:06:50,397 DEBUG [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:50,398 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(327): Store=cf5451ece9935ba018e98673baf4e95b/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:50,398 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,398 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,399 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,400 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,400 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,400 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T19:06:50,401 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,403 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:06:50,403 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened cf5451ece9935ba018e98673baf4e95b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10680689120, jitterRate=-0.0052833110094070435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T19:06:50,404 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:50,404 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for cf5451ece9935ba018e98673baf4e95b: Running coprocessor pre-open hook at 1731956810391Writing region info on filesystem at 1731956810391Initializing all the Stores at 1731956810391Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956810391Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956810392 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956810392Cleaning up temporary data from old regions at 1731956810400 (+8 ms)Running coprocessor post-open hooks at 1731956810404 (+4 ms)Region opened successfully at 1731956810404 2024-11-18T19:06:50,405 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., pid=79, masterSystemTime=1731956810387 2024-11-18T19:06:50,407 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:50,407 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:50,407 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:50,407 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => cd0db3f2ace15f2ccb5d44bb7a079f9c, NAME => 'table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-18T19:06:50,407 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:06:50,408 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,408 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:50,408 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,408 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,408 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=cf5451ece9935ba018e98673baf4e95b, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:50,409 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:50,410 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd0db3f2ace15f2ccb5d44bb7a079f9c columnFamilyName f 2024-11-18T19:06:50,411 DEBUG [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:50,411 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(327): Store=cd0db3f2ace15f2ccb5d44bb7a079f9c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:50,411 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=77 2024-11-18T19:06:50,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=77, state=SUCCESS, hasLock=false; OpenRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538 in 175 msec 2024-11-18T19:06:50,413 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd0db3f2ace15f2ccb5d44bb7a079f9c columnFamilyName f3 2024-11-18T19:06:50,413 DEBUG [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:50,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, ASSIGN in 331 msec 2024-11-18T19:06:50,413 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(327): Store=cd0db3f2ace15f2ccb5d44bb7a079f9c/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:50,413 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,414 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd0db3f2ace15f2ccb5d44bb7a079f9c columnFamilyName mob 2024-11-18T19:06:50,415 DEBUG [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:50,415 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(327): Store=cd0db3f2ace15f2ccb5d44bb7a079f9c/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:50,416 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,416 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,417 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,417 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,418 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,418 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T19:06:50,419 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,421 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:06:50,421 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened cd0db3f2ace15f2ccb5d44bb7a079f9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11795703200, jitterRate=0.09856046736240387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T19:06:50,421 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:50,422 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for cd0db3f2ace15f2ccb5d44bb7a079f9c: Running coprocessor pre-open hook at 1731956810408Writing region info on filesystem at 1731956810408Initializing all the Stores at 1731956810409 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956810409Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956810409Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956810409Cleaning up temporary data from old regions at 1731956810418 (+9 ms)Running coprocessor post-open hooks at 1731956810421 (+3 ms)Region opened successfully at 1731956810422 (+1 ms) 2024-11-18T19:06:50,422 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c., pid=80, masterSystemTime=1731956810387 2024-11-18T19:06:50,424 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:50,424 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:50,424 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=cd0db3f2ace15f2ccb5d44bb7a079f9c, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:50,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:50,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=78 2024-11-18T19:06:50,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538 in 190 msec 2024-11-18T19:06:50,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=76 2024-11-18T19:06:50,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=76, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, ASSIGN in 347 msec 2024-11-18T19:06:50,430 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956810430"}]},"ts":"1731956810430"} 2024-11-18T19:06:50,431 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-18T19:06:50,431 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'table1' completed 2024-11-18T19:06:50,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) in 1.2230 sec 2024-11-18T19:06:50,666 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-18T19:06:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T19:06:51,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: default:table1 completed 2024-11-18T19:06:51,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:51,367 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3,hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1 from hbase tables test-1731956706111 to tables table1 2024-11-18T19:06:51,367 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1731956706111 into table1 2024-11-18T19:06:51,369 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367 from hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3,hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1 2024-11-18T19:06:51,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400b89bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:51,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:51,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:51,371 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:51,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:51,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:51,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2014bf25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:51,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:51,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:51,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:51,373 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:51,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f19664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:51,373 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:51,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:51,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61d2ba1c 2024-11-18T19:06:51,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:51,375 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:51,376 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-18T19:06:51,376 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-18T19:06:51,376 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:51,376 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:51,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:51,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:51,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:51,379 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-18T19:06:51,379 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-18T19:06:51,379 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-18T19:06:51,379 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/partitions_4b228527-a292-4981-a541-7e9e6b764e82 2024-11-18T19:06:51,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741985_1161 (size=177) 2024-11-18T19:06:51,786 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:51,786 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:51,786 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,764 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-10803406642584034048.jar 2024-11-18T19:06:52,765 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,765 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,765 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,766 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:52,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:52,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:52,767 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:52,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:52,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:52,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:52,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:52,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:52,768 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:52,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:52,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,769 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,770 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,770 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:52,770 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:52,770 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-18T19:06:52,770 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:52,770 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:52,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:52,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:52,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:52,771 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-18T19:06:52,771 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:52,783 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:52,796 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-18T19:06:52,891 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1393136616_0014_be79c3fe-df9e-4086-b33c-0172f13daa40/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:52,891 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1393136616_0014_be79c3fe-df9e-4086-b33c-0172f13daa40/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:52,952 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/c51bc94ac86f4f448c46a9da09c3d19c 2024-11-18T19:06:52,955 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:53,003 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b77fe233b3544d71b762e205f6df7bad 2024-11-18T19:06:53,006 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:53,042 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f/b697bbeabff04e5e8e1b821b7421fa7d 2024-11-18T19:06:53,046 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:53,088 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/fa1be2101965d7b06c7ddc60e0eee0f1/mob/d41d8cd98f00b204e9800998ecf8427e2024111861588bf21f584367b581b2025850eb35_b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:06:53,091 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:53,125 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956750265/default/test-1731956706111/archive/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/mob/279d7b241f8c4edd9070df580f246dc0 2024-11-18T19:06:53,128 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:53,160 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aafe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:53,160 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:53,160 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:53,161 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:53,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:53,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:53,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@142758b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:53,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:53,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:53,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:53,163 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:53,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5a0798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:53,164 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:53,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:53,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:53,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:53,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1731956706111-1-0', locateType=CURRENT is [region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:53,168 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-18T19:06:53,169 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:53,169 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:53,169 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:53,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:53,169 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731956706111-1-0] 2024-11-18T19:06:53,169 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:53,176 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e1b8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:53,176 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:53,176 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:53,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:53,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:53,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:53,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@252bd088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:53,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:53,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:53,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:53,178 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:53,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550d0061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:53,179 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:53,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:53,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:53,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:53,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1731956706111-3-0', locateType=CURRENT is [region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:53,182 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-18T19:06:53,182 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:53,182 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:53,182 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:53,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:53,182 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731956706111-3-0] 2024-11-18T19:06:53,182 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:53,188 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/_temporary/0/_temporary/attempt_local1393136616_0014_r_000000_0/f/430f2b7713e041ff927cd90f09b52bd5 is 59, key is row-test-1731956706111-1-10/f:q1/1731956734874/Put/seqid=0 2024-11-18T19:06:53,189 WARN [Thread-1806 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:53,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741986_1162 (size=11125) 2024-11-18T19:06:53,593 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/_temporary/0/_temporary/attempt_local1393136616_0014_r_000000_0/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 is 192, key is row-test-1731956706111-3-100/mob:q1/1731956735228/Put/seqid=0 2024-11-18T19:06:53,595 WARN [Thread-1808 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:53,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741987_1163 (size=13404) 2024-11-18T19:06:54,022 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70dd02a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,022 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:54,022 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:54,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:54,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:54,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:54,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7952e921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:54,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:54,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:54,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:54,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf8f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,026 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:54,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:54,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:54,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:54,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:54,030 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-18T19:06:54,030 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:54,031 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:54,031 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:54,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:54,031 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-18T19:06:54,031 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:54,035 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/_temporary/0/_temporary/attempt_local1393136616_0014_r_000001_0/f/b77e9fd68b4b40949084bdb703491436 is 37, key is row10/f:q1/1731956708812/Put/seqid=0 2024-11-18T19:06:54,037 WARN [Thread-1823 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:54,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741988_1164 (size=8742) 2024-11-18T19:06:54,933 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367 2024-11-18T19:06:54,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4efd3712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:54,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:54,935 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:54,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:54,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:54,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae0f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:54,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:54,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:54,937 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:54,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f714187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:54,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:54,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:54,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:54,942 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/_SUCCESS 2024-11-18T19:06:54,949 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:54,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:54,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58981d6f 2024-11-18T19:06:54,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:54,951 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:54,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:54,955 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:54,960 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/f/430f2b7713e041ff927cd90f09b52bd5 first=Optional[row-test-1731956706111-1-0] last=Optional[row-test-1731956706111-1-98] 2024-11-18T19:06:54,963 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/f/b77e9fd68b4b40949084bdb703491436 first=Optional[row0] last=Optional[row98] 2024-11-18T19:06:54,965 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 first=Optional[row-test-1731956706111-3-0] last=Optional[row-test-1731956706111-3-99] 2024-11-18T19:06:54,969 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/f/b77e9fd68b4b40949084bdb703491436 for inclusion in cd0db3f2ace15f2ccb5d44bb7a079f9c/f 2024-11-18T19:06:54,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/f/430f2b7713e041ff927cd90f09b52bd5 for inclusion in cf5451ece9935ba018e98673baf4e95b/f 2024-11-18T19:06:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row-test-1731956706111-1-0 last=row-test-1731956706111-1-98 2024-11-18T19:06:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-18T19:06:54,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 for inclusion in cf5451ece9935ba018e98673baf4e95b/mob 2024-11-18T19:06:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-18T19:06:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-11-18T19:06:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HRegion(2603): Flush status journal for cd0db3f2ace15f2ccb5d44bb7a079f9c: 2024-11-18T19:06:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/f/b77e9fd68b4b40949084bdb703491436 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/b77e9fd68b4b40949084bdb703491436 2024-11-18T19:06:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row-test-1731956706111-3-0 last=row-test-1731956706111-3-99 2024-11-18T19:06:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-18T19:06:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegion(2603): Flush status journal for cf5451ece9935ba018e98673baf4e95b: 2024-11-18T19:06:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/f/430f2b7713e041ff927cd90f09b52bd5 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/430f2b7713e041ff927cd90f09b52bd5 2024-11-18T19:06:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/b77e9fd68b4b40949084bdb703491436 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c/f/49f2b42722cf4b4b81da36a812b65056_SeqId_4_ 2024-11-18T19:06:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2780ef2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:54,979 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:54,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:54,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:54,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480d33ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:54,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/430f2b7713e041ff927cd90f09b52bd5 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f/11358a23222d46b883f1e15799c5b2a4_SeqId_4_ 2024-11-18T19:06:54,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956811367/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 2024-11-18T19:06:54,980 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:54,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd6d77b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:54,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:54,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7222b730 2024-11-18T19:06:54,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/mob/449217a68b274db6a7de94fa56ed6526_SeqId_4_ 2024-11-18T19:06:54,983 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aad0ea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:54,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:54,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:54,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:54,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65addd00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:54,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:54,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:54,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:54,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:54,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:54,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:54,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369aee1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:54,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:54,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59c47338 2024-11-18T19:06:54,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:54,989 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:54,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:54,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:54,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:54,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:54,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:54,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:54,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:55,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643. 2024-11-18T19:06:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:55,003 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/b77e9fd68b4b40949084bdb703491436 into cd0db3f2ace15f2ccb5d44bb7a079f9c/f as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c/f/49f2b42722cf4b4b81da36a812b65056_SeqId_4_ - updating store file list. 2024-11-18T19:06:55,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:55,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:55,008 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c/f/49f2b42722cf4b4b81da36a812b65056_SeqId_4_ into cd0db3f2ace15f2ccb5d44bb7a079f9c/f 2024-11-18T19:06:55,008 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/b77e9fd68b4b40949084bdb703491436 into cd0db3f2ace15f2ccb5d44bb7a079f9c/f (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c/f/49f2b42722cf4b4b81da36a812b65056_SeqId_4_) 2024-11-18T19:06:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/b77e9fd68b4b40949084bdb703491436 2024-11-18T19:06:55,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@650ca8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b42bea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:55,011 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:55,012 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c37ccd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4081aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:55,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:55,013 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:55,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b2b9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,013 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:55,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a209df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:55,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:55,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f65f463 2024-11-18T19:06:55,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:55,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:55,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45a35f49 2024-11-18T19:06:55,015 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-18T19:06:55,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:55,016 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:55,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:55,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:55,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-18T19:06:55,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:55,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:55,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:55,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:55,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:55,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:55,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:55,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:55,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643. 2024-11-18T19:06:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:55,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:55,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:55,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/430f2b7713e041ff927cd90f09b52bd5 into cf5451ece9935ba018e98673baf4e95b/f as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f/11358a23222d46b883f1e15799c5b2a4_SeqId_4_ - updating store file list. 2024-11-18T19:06:55,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f/11358a23222d46b883f1e15799c5b2a4_SeqId_4_ into cf5451ece9935ba018e98673baf4e95b/f 2024-11-18T19:06:55,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/430f2b7713e041ff927cd90f09b52bd5 into cf5451ece9935ba018e98673baf4e95b/f (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f/11358a23222d46b883f1e15799c5b2a4_SeqId_4_) 2024-11-18T19:06:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/f/430f2b7713e041ff927cd90f09b52bd5 2024-11-18T19:06:55,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 into cf5451ece9935ba018e98673baf4e95b/mob as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/mob/449217a68b274db6a7de94fa56ed6526_SeqId_4_ - updating store file list. 2024-11-18T19:06:55,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/mob/449217a68b274db6a7de94fa56ed6526_SeqId_4_ into cf5451ece9935ba018e98673baf4e95b/mob 2024-11-18T19:06:55,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 into cf5451ece9935ba018e98673baf4e95b/mob (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/mob/449217a68b274db6a7de94fa56ed6526_SeqId_4_) 2024-11-18T19:06:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__iil59dh3msr71vkh8pbpgco3dhet0679v98778h53a3f1tguaghqkb3gfbem9u1e/mob/90cd5be7cc0a4b1ebe54fce4f9f8a570 2024-11-18T19:06:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d0acb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:55,053 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:55,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:55,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:55,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79ff4b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:55,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:55,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,055 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:55,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a3dcf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:55,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:55,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4e2beab1 2024-11-18T19:06:55,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:55,058 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-18T19:06:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:55,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:55,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:55,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-18T19:06:55,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:55,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:55,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:55,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:55,076 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:55,076 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:55,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:55,076 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:55,076 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-18T19:06:55,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=2 , the old value is region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:06:55,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:06:55,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=2 from cache 2024-11-18T19:06:55,077 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-18T19:06:55,103 INFO [Time-limited test {}] impl.RestoreTablesClient(181): Restoring 'test-1731956706111' to 'table1' from log dirs: hdfs://localhost:45363/backupUT/backup_1731956785190/default/test-1731956706111/f2/d6af71cf1d374f1d979dac1f181e260c 2024-11-18T19:06:55,107 DEBUG [Time-limited test {}] util.RestoreTool(177): Found descriptor 'test-1731956706111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} through backup_1731956785190 2024-11-18T19:06:55,109 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table table1 from 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=table1 2024-11-18T19:06:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741989_1165 (size=938) 2024-11-18T19:06:55,182 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-18T19:06:55,523 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/.tabledesc/.tableinfo.0000000001.938 2024-11-18T19:06:55,523 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/.tabledesc/.tableinfo.0000000002.938 2024-11-18T19:06:55,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=table1}] 2024-11-18T19:06:55,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, REOPEN/MOVE}, {pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, REOPEN/MOVE}] 2024-11-18T19:06:55,528 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, REOPEN/MOVE 2024-11-18T19:06:55,528 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, REOPEN/MOVE 2024-11-18T19:06:55,529 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=cf5451ece9935ba018e98673baf4e95b, regionState=CLOSING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:55,529 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=cd0db3f2ace15f2ccb5d44bb7a079f9c, regionState=CLOSING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:55,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, REOPEN/MOVE because future has completed 2024-11-18T19:06:55,531 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T19:06:55,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:55,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, REOPEN/MOVE because future has completed 2024-11-18T19:06:55,532 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T19:06:55,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:55,684 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(122): Close cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:55,684 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T19:06:55,684 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1722): Closing cf5451ece9935ba018e98673baf4e95b, disabling compactions & flushes 2024-11-18T19:06:55,684 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1755): Closing region table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:55,684 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:55,684 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1843): Acquired close lock on table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. after waiting 0 ms 2024-11-18T19:06:55,684 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1853): Updates disabled for region table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:55,688 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-18T19:06:55,689 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:06:55,689 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1973): Closed table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:55,689 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1676): Region close journal for cf5451ece9935ba018e98673baf4e95b: Waiting for close lock at 1731956815684Running coprocessor pre-close hooks at 1731956815684Disabling compacts and flushes for region at 1731956815684Disabling writes for close at 1731956815684Writing region close event to WAL at 1731956815685 (+1 ms)Running coprocessor post-close hooks at 1731956815689 (+4 ms)Closed at 1731956815689 2024-11-18T19:06:55,689 WARN [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegionServer(3299): Not adding moved region record: cf5451ece9935ba018e98673baf4e95b to self. 2024-11-18T19:06:55,691 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(157): Closed cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:55,691 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:55,691 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T19:06:55,691 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing cd0db3f2ace15f2ccb5d44bb7a079f9c, disabling compactions & flushes 2024-11-18T19:06:55,691 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:55,691 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:55,691 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. after waiting 0 ms 2024-11-18T19:06:55,691 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:55,691 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=cf5451ece9935ba018e98673baf4e95b, regionState=CLOSED 2024-11-18T19:06:55,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:55,695 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-18T19:06:55,695 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:06:55,696 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:55,696 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for cd0db3f2ace15f2ccb5d44bb7a079f9c: Waiting for close lock at 1731956815691Running coprocessor pre-close hooks at 1731956815691Disabling compacts and flushes for region at 1731956815691Disabling writes for close at 1731956815691Writing region close event to WAL at 1731956815692 (+1 ms)Running coprocessor post-close hooks at 1731956815695 (+3 ms)Closed at 1731956815696 (+1 ms) 2024-11-18T19:06:55,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=83 2024-11-18T19:06:55,696 WARN [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: cd0db3f2ace15f2ccb5d44bb7a079f9c to self. 2024-11-18T19:06:55,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=83, state=SUCCESS, hasLock=false; CloseRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538 in 162 msec 2024-11-18T19:06:55,697 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, REOPEN/MOVE; state=CLOSED, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=true 2024-11-18T19:06:55,697 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:55,698 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=cd0db3f2ace15f2ccb5d44bb7a079f9c, regionState=CLOSED 2024-11-18T19:06:55,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:55,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-18T19:06:55,702 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, REOPEN/MOVE; state=CLOSED, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=true 2024-11-18T19:06:55,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538 in 169 msec 2024-11-18T19:06:55,755 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T19:06:55,847 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=cd0db3f2ace15f2ccb5d44bb7a079f9c, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:55,847 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=cf5451ece9935ba018e98673baf4e95b, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:55,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, REOPEN/MOVE because future has completed 2024-11-18T19:06:55,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:55,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, REOPEN/MOVE because future has completed 2024-11-18T19:06:55,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:55,892 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-18T19:06:56,004 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:56,005 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => cd0db3f2ace15f2ccb5d44bb7a079f9c, NAME => 'table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-18T19:06:56,005 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:06:56,005 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,005 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:56,005 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,005 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,007 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,007 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd0db3f2ace15f2ccb5d44bb7a079f9c columnFamilyName f 2024-11-18T19:06:56,008 DEBUG [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:56,013 DEBUG [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c/f/49f2b42722cf4b4b81da36a812b65056_SeqId_4_ 2024-11-18T19:06:56,013 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(327): Store=cd0db3f2ace15f2ccb5d44bb7a079f9c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:56,013 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,014 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd0db3f2ace15f2ccb5d44bb7a079f9c columnFamilyName f2 2024-11-18T19:06:56,015 DEBUG [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:56,015 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(327): Store=cd0db3f2ace15f2ccb5d44bb7a079f9c/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:56,015 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,016 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd0db3f2ace15f2ccb5d44bb7a079f9c columnFamilyName mob 2024-11-18T19:06:56,016 DEBUG [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:56,016 INFO [StoreOpener-cd0db3f2ace15f2ccb5d44bb7a079f9c-1 {}] regionserver.HStore(327): Store=cd0db3f2ace15f2ccb5d44bb7a079f9c/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:56,017 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,017 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,018 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,019 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,019 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,020 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T19:06:56,021 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,022 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened cd0db3f2ace15f2ccb5d44bb7a079f9c; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9855944640, jitterRate=-0.08209362626075745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T19:06:56,022 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:06:56,022 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for cd0db3f2ace15f2ccb5d44bb7a079f9c: Running coprocessor pre-open hook at 1731956816006Writing region info on filesystem at 1731956816006Initializing all the Stores at 1731956816006Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956816006Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956816006Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956816006Cleaning up temporary data from old regions at 1731956816019 (+13 ms)Running coprocessor post-open hooks at 1731956816022 (+3 ms)Region opened successfully at 1731956816022 2024-11-18T19:06:56,023 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c., pid=88, masterSystemTime=1731956816001 2024-11-18T19:06:56,025 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:56,025 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:06:56,025 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:56,025 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => cf5451ece9935ba018e98673baf4e95b, NAME => 'table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-18T19:06:56,025 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=cd0db3f2ace15f2ccb5d44bb7a079f9c, regionState=OPEN, openSeqNum=7, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:56,026 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:06:56,026 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,026 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:56,026 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,026 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,027 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:56,028 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5451ece9935ba018e98673baf4e95b columnFamilyName f 2024-11-18T19:06:56,028 DEBUG [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:56,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=84 2024-11-18T19:06:56,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure cd0db3f2ace15f2ccb5d44bb7a079f9c, server=818bff808382,33643,1731956698538 in 178 msec 2024-11-18T19:06:56,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=cd0db3f2ace15f2ccb5d44bb7a079f9c, REOPEN/MOVE in 503 msec 2024-11-18T19:06:56,034 DEBUG [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f/11358a23222d46b883f1e15799c5b2a4_SeqId_4_ 2024-11-18T19:06:56,034 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(327): Store=cf5451ece9935ba018e98673baf4e95b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:56,034 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,035 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5451ece9935ba018e98673baf4e95b columnFamilyName f2 2024-11-18T19:06:56,035 DEBUG [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:56,036 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(327): Store=cf5451ece9935ba018e98673baf4e95b/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:56,036 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,036 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf5451ece9935ba018e98673baf4e95b columnFamilyName mob 2024-11-18T19:06:56,037 DEBUG [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:06:56,041 DEBUG [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/mob/449217a68b274db6a7de94fa56ed6526_SeqId_4_ 2024-11-18T19:06:56,041 INFO [StoreOpener-cf5451ece9935ba018e98673baf4e95b-1 {}] regionserver.HStore(327): Store=cf5451ece9935ba018e98673baf4e95b/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:06:56,042 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,042 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,043 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,044 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,044 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,045 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-18T19:06:56,046 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,047 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened cf5451ece9935ba018e98673baf4e95b; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10636770880, jitterRate=-0.009373515844345093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-18T19:06:56,047 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:06:56,047 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for cf5451ece9935ba018e98673baf4e95b: Running coprocessor pre-open hook at 1731956816026Writing region info on filesystem at 1731956816026Initializing all the Stores at 1731956816027 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956816027Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956816027Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956816027Cleaning up temporary data from old regions at 1731956816044 (+17 ms)Running coprocessor post-open hooks at 1731956816047 (+3 ms)Region opened successfully at 1731956816047 2024-11-18T19:06:56,048 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., pid=87, masterSystemTime=1731956816001 2024-11-18T19:06:56,050 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:56,050 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:06:56,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=cf5451ece9935ba018e98673baf4e95b, regionState=OPEN, openSeqNum=7, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:56,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:56,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=83 2024-11-18T19:06:56,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=83, state=SUCCESS, hasLock=false; OpenRegionProcedure cf5451ece9935ba018e98673baf4e95b, server=818bff808382,33643,1731956698538 in 204 msec 2024-11-18T19:06:56,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-18T19:06:56,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=cf5451ece9935ba018e98673baf4e95b, REOPEN/MOVE in 528 msec 2024-11-18T19:06:56,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-18T19:06:56,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=table1 in 532 msec 2024-11-18T19:06:56,060 DEBUG [PEWorker-1 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=table1 2024-11-18T19:06:56,061 DEBUG [PEWorker-1 {}] backup.HFileArchiver(265): No files to dispose of in table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., family=f3 2024-11-18T19:06:56,062 DEBUG [PEWorker-1 {}] backup.HFileArchiver(265): No files to dispose of in table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c., family=f3 2024-11-18T19:06:56,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; ModifyTableProcedure table=table1 in 952 msec 2024-11-18T19:06:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-18T19:06:56,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:table1 completed 2024-11-18T19:06:56,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:56,069 INFO [Time-limited test {}] util.RestoreTool(200): Changed table1 to: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:06:56,069 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore incremental backup from directory hdfs://localhost:45363/backupUT/backup_1731956785190/default/test-1731956706111/f2/d6af71cf1d374f1d979dac1f181e260c from hbase tables test-1731956706111 to tables table1 2024-11-18T19:06:56,069 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1731956706111 into table1 2024-11-18T19:06:56,070 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956816069 from hdfs://localhost:45363/backupUT/backup_1731956785190/default/test-1731956706111/f2/d6af71cf1d374f1d979dac1f181e260c 2024-11-18T19:06:56,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9fdab7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:56,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:56,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:56,072 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:56,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:56,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:56,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b31c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:56,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:56,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:56,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:56,074 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:56,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa3c005, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:56,074 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:56,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:56,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1ecb95b4 2024-11-18T19:06:56,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:56,076 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:56,077 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-18T19:06:56,077 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-18T19:06:56,077 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:56,077 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:56,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:56,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:56,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:56,081 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-18T19:06:56,081 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-18T19:06:56,081 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-18T19:06:56,081 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/partitions_ea1de8e1-c9e8-4a4e-b4ba-06055cb2ea93 2024-11-18T19:06:56,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741990_1166 (size=177) 2024-11-18T19:06:56,488 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:56,488 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:56,488 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,468 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-859620901718900501.jar 2024-11-18T19:06:57,468 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,469 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,469 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,469 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,469 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,469 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,470 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,470 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,470 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,470 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:57,470 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:06:57,470 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:06:57,471 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:06:57,471 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:06:57,471 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:06:57,471 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:06:57,471 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:06:57,471 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:06:57,472 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:06:57,472 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:06:57,472 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,472 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,473 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,473 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,473 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,473 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:06:57,473 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:06:57,474 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-18T19:06:57,474 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:57,474 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:57,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:57,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:57,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:57,474 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-18T19:06:57,474 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:06:57,486 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:06:57,499 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-18T19:06:57,591 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local394614945_0015_4a8332da-0376-4ad3-80e5-347baa3b3916/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:06:57,591 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local394614945_0015_4a8332da-0376-4ad3-80e5-347baa3b3916/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:06:57,662 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956785190/default/test-1731956706111/f2/d6af71cf1d374f1d979dac1f181e260c 2024-11-18T19:06:57,665 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:06:57,704 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14fcafa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:57,704 DEBUG [pool-281-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:57,704 DEBUG [pool-281-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:57,705 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:57,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:57,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:57,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ca64ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:57,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:57,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:57,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:57,707 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:57,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a387d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:57,708 DEBUG [pool-281-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:57,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:57,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:57,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:57,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1731956706111-2-0', locateType=CURRENT is [region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=7] 2024-11-18T19:06:57,712 INFO [pool-281-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-281-thread-1. 2024-11-18T19:06:57,712 DEBUG [pool-281-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:57,712 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:57,713 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:57,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:57,713 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731956706111-2-0] 2024-11-18T19:06:57,713 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:06:57,716 DEBUG [pool-281-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956816069/_temporary/0/_temporary/attempt_local394614945_0015_r_000000_0/f2/d07d44a4d5624acc8c88fc0802f16c2b is 58, key is row-test-1731956706111-2-0/f2:q1/1731956780182/Put/seqid=0 2024-11-18T19:06:57,718 WARN [Thread-1887 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:06:57,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741991_1167 (size=5608) 2024-11-18T19:06:58,634 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956816069 2024-11-18T19:06:58,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b585f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:58,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:58,636 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:58,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:58,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:58,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc8bfbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:58,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:58,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,638 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:58,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3645386a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,639 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:58,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:58,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:58,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:58,644 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956816069/_SUCCESS 2024-11-18T19:06:58,647 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:58,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:58,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@19576976 2024-11-18T19:06:58,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:58,649 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:58,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=7] 2024-11-18T19:06:58,654 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-18T19:06:58,660 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956816069/f2/d07d44a4d5624acc8c88fc0802f16c2b first=Optional[row-test-1731956706111-2-0] last=Optional[row-test-1731956706111-2-6] 2024-11-18T19:06:58,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956816069/f2/d07d44a4d5624acc8c88fc0802f16c2b for inclusion in cf5451ece9935ba018e98673baf4e95b/f2 2024-11-18T19:06:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row-test-1731956706111-2-0 last=row-test-1731956706111-2-6 2024-11-18T19:06:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-18T19:06:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegion(2603): Flush status journal for cf5451ece9935ba018e98673baf4e95b: 2024-11-18T19:06:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-default-table1-1731956816069/f2/d07d44a4d5624acc8c88fc0802f16c2b to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__s07spq2n2llphpunj1ci4kmc2ikmnful504j28oij8l1sgvtt98ntl56c84rb0k9/f2/d07d44a4d5624acc8c88fc0802f16c2b 2024-11-18T19:06:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__s07spq2n2llphpunj1ci4kmc2ikmnful504j28oij8l1sgvtt98ntl56c84rb0k9/f2/d07d44a4d5624acc8c88fc0802f16c2b as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f2/03de5c07f29d48ec82f663389c309403_SeqId_9_ 2024-11-18T19:06:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edce354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:58,671 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:58,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:58,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:58,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325a3cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:58,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:58,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:58,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674d4a49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:58,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:58,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@11145061 2024-11-18T19:06:58,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:58,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:06:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:58,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:58,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:58,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:06:58,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:58,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:58,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:58,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:58,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__s07spq2n2llphpunj1ci4kmc2ikmnful504j28oij8l1sgvtt98ntl56c84rb0k9/f2/d07d44a4d5624acc8c88fc0802f16c2b into cf5451ece9935ba018e98673baf4e95b/f2 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f2/03de5c07f29d48ec82f663389c309403_SeqId_9_ - updating store file list. 2024-11-18T19:06:58,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f2/03de5c07f29d48ec82f663389c309403_SeqId_9_ into cf5451ece9935ba018e98673baf4e95b/f2 2024-11-18T19:06:58,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__s07spq2n2llphpunj1ci4kmc2ikmnful504j28oij8l1sgvtt98ntl56c84rb0k9/f2/d07d44a4d5624acc8c88fc0802f16c2b into cf5451ece9935ba018e98673baf4e95b/f2 (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/f2/03de5c07f29d48ec82f663389c309403_SeqId_9_) 2024-11-18T19:06:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__table1__s07spq2n2llphpunj1ci4kmc2ikmnful504j28oij8l1sgvtt98ntl56c84rb0k9/f2/d07d44a4d5624acc8c88fc0802f16c2b 2024-11-18T19:06:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54967b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:06:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:06:58,701 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:06:58,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:06:58,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:06:58,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c59963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:06:58,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:06:58,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,702 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:06:58,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399a4bdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:06:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:06:58,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:06:58,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f2347a 2024-11-18T19:06:58,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:06:58,704 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-18T19:06:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:06:58,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:06:58,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:06:58,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-18T19:06:58,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:06:58,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:06:58,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:06:58,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:06:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:06:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:58,723 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:06:58,723 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:06:58,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:06:58,723 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-18T19:06:58,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=7 , the old value is region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=7, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:06:58,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=7 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:06:58,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=7 from cache 2024-11-18T19:06:58,723 INFO [Time-limited test {}] impl.RestoreTablesClient(187): test-1731956706111 has been successfully restored to table1 2024-11-18T19:06:58,723 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-18T19:06:58,723 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1731956785190 hdfs://localhost:45363/backupUT/backup_1731956785190/default/test-1731956706111/ 2024-11-18T19:06:58,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:06:58,724 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:45363/backupUT/backup_1731956750265/.backup.manifest 2024-11-18T19:06:58,724 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-18T19:06:58,726 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731956750265/.backup.manifest 2024-11-18T19:06:58,726 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17319567061111' to 'ns2:table2' from full backup image hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111 2024-11-18T19:06:58,728 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'ns2:test-17319567061111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1731956785190 2024-11-18T19:06:58,731 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'ns2:table2', preserving region splits 2024-11-18T19:06:58,731 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable ns2:table2 2024-11-18T19:06:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=ns2:table2 2024-11-18T19:06:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T19:06:58,735 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956818735"}]},"ts":"1731956818735"} 2024-11-18T19:06:58,736 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2024-11-18T19:06:58,736 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set ns2:table2 to state=DISABLING 2024-11-18T19:06:58,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=ns2:table2}] 2024-11-18T19:06:58,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, UNASSIGN}] 2024-11-18T19:06:58,739 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, UNASSIGN 2024-11-18T19:06:58,739 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=91 updating hbase:meta row=7cca13c6ec211bff31d3fc7f72705603, regionState=CLOSING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:06:58,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, UNASSIGN because future has completed 2024-11-18T19:06:58,741 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T19:06:58,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cca13c6ec211bff31d3fc7f72705603, server=818bff808382,33643,1731956698538}] 2024-11-18T19:06:58,833 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-18T19:06:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T19:06:58,893 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(122): Close 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:58,893 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T19:06:58,893 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1722): Closing 7cca13c6ec211bff31d3fc7f72705603, disabling compactions & flushes 2024-11-18T19:06:58,893 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1755): Closing region ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:58,894 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:58,894 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. after waiting 0 ms 2024-11-18T19:06:58,894 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:58,898 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-18T19:06:58,898 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:06:58,898 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1973): Closed ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. 2024-11-18T19:06:58,898 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1676): Region close journal for 7cca13c6ec211bff31d3fc7f72705603: Waiting for close lock at 1731956818893Running coprocessor pre-close hooks at 1731956818893Disabling compacts and flushes for region at 1731956818893Disabling writes for close at 1731956818894 (+1 ms)Writing region close event to WAL at 1731956818894Running coprocessor post-close hooks at 1731956818898 (+4 ms)Closed at 1731956818898 2024-11-18T19:06:58,900 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(157): Closed 7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:58,901 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=91 updating hbase:meta row=7cca13c6ec211bff31d3fc7f72705603, regionState=CLOSED 2024-11-18T19:06:58,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cca13c6ec211bff31d3fc7f72705603, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:06:58,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-18T19:06:58,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; CloseRegionProcedure 7cca13c6ec211bff31d3fc7f72705603, server=818bff808382,33643,1731956698538 in 162 msec 2024-11-18T19:06:58,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-18T19:06:58,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7cca13c6ec211bff31d3fc7f72705603, UNASSIGN in 166 msec 2024-11-18T19:06:58,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-18T19:06:58,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=ns2:table2 in 169 msec 2024-11-18T19:06:58,909 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956818909"}]},"ts":"1731956818909"} 2024-11-18T19:06:58,910 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2024-11-18T19:06:58,910 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set ns2:table2 to state=DISABLED 2024-11-18T19:06:58,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; DisableTableProcedure table=ns2:table2 in 180 msec 2024-11-18T19:06:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T19:06:59,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: ns2:table2 completed 2024-11-18T19:06:59,052 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.2 truncate ns2:table2 2024-11-18T19:06:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) 2024-11-18T19:06:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T19:06:59,054 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:table2' regions in transition 2024-11-18T19:06:59,057 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:59,059 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/f, FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/recovered.edits] 2024-11-18T19:06:59,063 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/f/7575ecb710b64383be65dd5eb14e741f_SeqId_4_ to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/archive/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/f/7575ecb710b64383be65dd5eb14e741f_SeqId_4_ 2024-11-18T19:06:59,065 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/recovered.edits/6.seqid to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/archive/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603/recovered.edits/6.seqid 2024-11-18T19:06:59,066 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/7cca13c6ec211bff31d3fc7f72705603 2024-11-18T19:06:59,066 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived ns2:table2 regions 2024-11-18T19:06:59,070 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2024-11-18T19:06:59,072 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'ns2:table2' descriptor. 2024-11-18T19:06:59,072 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'ns2:table2' from region states. 2024-11-18T19:06:59,073 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731956819072"}]},"ts":"9223372036854775807"} 2024-11-18T19:06:59,074 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T19:06:59,074 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7cca13c6ec211bff31d3fc7f72705603, NAME => 'ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T19:06:59,074 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'ns2:table2' as deleted. 2024-11-18T19:06:59,074 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731956819074"}]},"ts":"9223372036854775807"} 2024-11-18T19:06:59,076 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table ns2:table2 state from META 2024-11-18T19:06:59,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741992_1168 (size=351) 2024-11-18T19:06:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T19:06:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T19:06:59,484 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1c7d96a664e933b6430bd1488e8a2937, NAME => 'ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9 2024-11-18T19:06:59,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741993_1169 (size=37) 2024-11-18T19:06:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T19:06:59,891 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:06:59,891 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 1c7d96a664e933b6430bd1488e8a2937, disabling compactions & flushes 2024-11-18T19:06:59,891 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:06:59,891 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:06:59,891 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. after waiting 0 ms 2024-11-18T19:06:59,891 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:06:59,891 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:06:59,891 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1c7d96a664e933b6430bd1488e8a2937: Waiting for close lock at 1731956819891Disabling compacts and flushes for region at 1731956819891Disabling writes for close at 1731956819891Writing region close event to WAL at 1731956819891Closed at 1731956819891 2024-11-18T19:06:59,898 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1731956819898"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956819898"}]},"ts":"1731956819898"} 2024-11-18T19:06:59,900 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T19:06:59,901 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956819900"}]},"ts":"1731956819900"} 2024-11-18T19:06:59,902 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-11-18T19:06:59,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=1c7d96a664e933b6430bd1488e8a2937, ASSIGN}] 2024-11-18T19:06:59,903 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=1c7d96a664e933b6430bd1488e8a2937, ASSIGN 2024-11-18T19:06:59,904 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=1c7d96a664e933b6430bd1488e8a2937, ASSIGN; state=OFFLINE, location=818bff808382,33643,1731956698538; forceNewPlan=false, retain=false 2024-11-18T19:07:00,054 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=1c7d96a664e933b6430bd1488e8a2937, regionState=OPENING, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:07:00,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=1c7d96a664e933b6430bd1488e8a2937, ASSIGN because future has completed 2024-11-18T19:07:00,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7d96a664e933b6430bd1488e8a2937, server=818bff808382,33643,1731956698538}] 2024-11-18T19:07:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T19:07:00,211 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:07:00,211 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c7d96a664e933b6430bd1488e8a2937, NAME => 'ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937.', STARTKEY => '', ENDKEY => ''} 2024-11-18T19:07:00,211 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-18T19:07:00,211 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,211 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T19:07:00,212 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,212 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,213 INFO [StoreOpener-1c7d96a664e933b6430bd1488e8a2937-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,214 INFO [StoreOpener-1c7d96a664e933b6430bd1488e8a2937-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c7d96a664e933b6430bd1488e8a2937 columnFamilyName f 2024-11-18T19:07:00,214 DEBUG [StoreOpener-1c7d96a664e933b6430bd1488e8a2937-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:07:00,215 INFO [StoreOpener-1c7d96a664e933b6430bd1488e8a2937-1 {}] regionserver.HStore(327): Store=1c7d96a664e933b6430bd1488e8a2937/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T19:07:00,215 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,216 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,216 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,216 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,216 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,218 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,220 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/1c7d96a664e933b6430bd1488e8a2937/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T19:07:00,220 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 1c7d96a664e933b6430bd1488e8a2937; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9664536640, jitterRate=-0.0999198853969574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T19:07:00,220 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:00,221 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 1c7d96a664e933b6430bd1488e8a2937: Running coprocessor pre-open hook at 1731956820212Writing region info on filesystem at 1731956820212Initializing all the Stores at 1731956820212Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956820213 (+1 ms)Cleaning up temporary data from old regions at 1731956820216 (+3 ms)Running coprocessor post-open hooks at 1731956820220 (+4 ms)Region opened successfully at 1731956820221 (+1 ms) 2024-11-18T19:07:00,222 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937., pid=95, masterSystemTime=1731956820207 2024-11-18T19:07:00,223 DEBUG [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:07:00,223 INFO [RS_OPEN_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:07:00,224 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=1c7d96a664e933b6430bd1488e8a2937, regionState=OPEN, openSeqNum=2, regionLocation=818bff808382,33643,1731956698538 2024-11-18T19:07:00,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7d96a664e933b6430bd1488e8a2937, server=818bff808382,33643,1731956698538 because future has completed 2024-11-18T19:07:00,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-18T19:07:00,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c7d96a664e933b6430bd1488e8a2937, server=818bff808382,33643,1731956698538 in 170 msec 2024-11-18T19:07:00,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-18T19:07:00,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=1c7d96a664e933b6430bd1488e8a2937, ASSIGN in 326 msec 2024-11-18T19:07:00,230 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956820230"}]},"ts":"1731956820230"} 2024-11-18T19:07:00,231 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-11-18T19:07:00,231 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:table2' completed 2024-11-18T19:07:00,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) in 1.1780 sec 2024-11-18T19:07:01,100 DEBUG [regionserver/818bff808382:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-18T19:07:01,102 WARN [regionserver/818bff808382:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 18, running: 1 2024-11-18T19:07:01,190 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-18T19:07:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35649 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T19:07:01,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: ns2:table2 completed 2024-11-18T19:07:01,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-18T19:07:01,206 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb from hbase tables ns2:test-17319567061111 to tables ns2:table2 2024-11-18T19:07:01,206 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17319567061111 into ns2:table2 2024-11-18T19:07:01,207 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956821206 from hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:07:01,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a54f91a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:01,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:07:01,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:07:01,209 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:07:01,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:07:01,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:07:01,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8adb04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:01,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:07:01,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:07:01,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:01,211 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:07:01,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ba13e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:01,211 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:07:01,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:07:01,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@129786d4 2024-11-18T19:07:01,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:07:01,213 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:07:01,213 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-18T19:07:01,214 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-11-18T19:07:01,214 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-18T19:07:01,214 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:07:01,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:07:01,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:07:01,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:07:01,217 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-11-18T19:07:01,217 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-11-18T19:07:01,217 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/partitions_c753866c-43b4-4885-9edd-5276e1644386 2024-11-18T19:07:01,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741994_1170 (size=153) 2024-11-18T19:07:01,624 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:01,624 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:01,624 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:01,919 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-18T19:07:02,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop-16513287491030939225.jar 2024-11-18T19:07:02,602 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,603 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,603 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,603 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,603 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,603 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,603 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,604 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,604 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,604 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:07:02,604 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-18T19:07:02,604 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-18T19:07:02,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-18T19:07:02,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-18T19:07:02,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-18T19:07:02,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-18T19:07:02,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-18T19:07:02,605 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-18T19:07:02,606 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-18T19:07:02,606 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-18T19:07:02,606 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,606 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-18T19:07:02,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-18T19:07:02,607 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-11-18T19:07:02,608 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:07:02,608 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:07:02,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:02,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:02,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:07:02,608 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-18T19:07:02,608 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-18T19:07:02,621 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-18T19:07:02,634 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-18T19:07:02,727 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1721647255_0016_ee211f3f-7e9c-4f07-9a23-e85af0dd8926/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-18T19:07:02,727 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/mapreduce.cluster.local.dir/job_local1721647255_0016_ee211f3f-7e9c-4f07-9a23-e85af0dd8926/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-18T19:07:02,786 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/2bf819358c514f5bb172f1c243472408 2024-11-18T19:07:02,790 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:07:02,833 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:45363/backupUT/backup_1731956750265/ns2/test-17319567061111/archive/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/f/a5c01da247874f4a876ea5005d2e8711 2024-11-18T19:07:02,837 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-18T19:07:02,866 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c50f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:02,866 DEBUG [pool-286-thread-1 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:07:02,866 DEBUG [pool-286-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:07:02,867 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:07:02,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:07:02,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:07:02,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84d2fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:02,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:07:02,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:07:02,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:02,869 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:07:02,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb0aec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:02,870 DEBUG [pool-286-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:07:02,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:07:02,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:07:02,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:07:02,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row-t20', locateType=CURRENT is [region=ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:07:02,874 INFO [pool-286-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-286-thread-1. 2024-11-18T19:07:02,874 DEBUG [pool-286-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:07:02,874 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:02,874 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:02,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:07:02,874 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-18T19:07:02,874 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 818bff808382 2024-11-18T19:07:02,879 DEBUG [pool-286-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956821206/_temporary/0/_temporary/attempt_local1721647255_0016_r_000000_0/f/e6983c0809be49d1b33c014cdf941a51 is 38, key is row-t20/f:q1/1731956735270/Put/seqid=0 2024-11-18T19:07:02,880 WARN [Thread-1962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [818bff808382:33643] Specified favored nodes: [818bff808382:33643] 2024-11-18T19:07:02,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741995_1171 (size=8935) 2024-11-18T19:07:03,769 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956821206 2024-11-18T19:07:03,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42050a34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:07:03,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:07:03,771 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:07:03,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:07:03,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:07:03,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f4c05d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:07:03,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:07:03,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,773 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:07:03,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70702c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,774 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:07:03,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:07:03,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:07:03,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:07:03,778 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956821206/_SUCCESS 2024-11-18T19:07:03,781 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:07:03,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:07:03,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68d1086d 2024-11-18T19:07:03,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:07:03,783 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:07:03,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:07:03,788 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-18T19:07:03,793 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956821206/f/e6983c0809be49d1b33c014cdf941a51 first=Optional[row-t20] last=Optional[row98] 2024-11-18T19:07:03,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956821206/f/e6983c0809be49d1b33c014cdf941a51 for inclusion in 1c7d96a664e933b6430bd1488e8a2937/f 2024-11-18T19:07:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(626): HFile bounds: first=row-t20 last=row98 2024-11-18T19:07:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-18T19:07:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegion(2603): Flush status journal for 1c7d96a664e933b6430bd1488e8a2937: 2024-11-18T19:07:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/hbase-staging/bulk_output-ns2-table2-1731956821206/f/e6983c0809be49d1b33c014cdf941a51 to hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__mhvkikgnucq1nh5i2jknabk8jolnik2safq9v7haq0hhm8mk5g8eeag4apr2cj6e/f/e6983c0809be49d1b33c014cdf941a51 2024-11-18T19:07:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__mhvkikgnucq1nh5i2jknabk8jolnik2safq9v7haq0hhm8mk5g8eeag4apr2cj6e/f/e6983c0809be49d1b33c014cdf941a51 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/1c7d96a664e933b6430bd1488e8a2937/f/318230b901344be699be00216527673f_SeqId_4_ 2024-11-18T19:07:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d9fa77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:07:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:07:03,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:07:03,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:07:03,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:07:03,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a6959ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:07:03,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:07:03,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:07:03,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7653d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:07:03,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:07:03,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ba427a9 2024-11-18T19:07:03,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:07:03,809 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T19:07:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:07:03,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:07:03,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:07:03,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T19:07:03,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:07:03,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:07:03,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:07:03,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:07:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:07:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:07:03,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__mhvkikgnucq1nh5i2jknabk8jolnik2safq9v7haq0hhm8mk5g8eeag4apr2cj6e/f/e6983c0809be49d1b33c014cdf941a51 into 1c7d96a664e933b6430bd1488e8a2937/f as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/1c7d96a664e933b6430bd1488e8a2937/f/318230b901344be699be00216527673f_SeqId_4_ - updating store file list. 2024-11-18T19:07:03,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/1c7d96a664e933b6430bd1488e8a2937/f/318230b901344be699be00216527673f_SeqId_4_ into 1c7d96a664e933b6430bd1488e8a2937/f 2024-11-18T19:07:03,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__mhvkikgnucq1nh5i2jknabk8jolnik2safq9v7haq0hhm8mk5g8eeag4apr2cj6e/f/e6983c0809be49d1b33c014cdf941a51 into 1c7d96a664e933b6430bd1488e8a2937/f (new location: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/1c7d96a664e933b6430bd1488e8a2937/f/318230b901344be699be00216527673f_SeqId_4_) 2024-11-18T19:07:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/staging/jenkins__ns2_table2__mhvkikgnucq1nh5i2jknabk8jolnik2safq9v7haq0hhm8mk5g8eeag4apr2cj6e/f/e6983c0809be49d1b33c014cdf941a51 2024-11-18T19:07:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d897d28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ClusterIdFetcher(90): Going to request 818bff808382,35649,-1 for getting cluster id 2024-11-18T19:07:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T19:07:03,834 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6e087e80-4df5-4144-b1b9-0d02cb0c3509' 2024-11-18T19:07:03,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T19:07:03,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6e087e80-4df5-4144-b1b9-0d02cb0c3509" 2024-11-18T19:07:03,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bba53f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [818bff808382,35649,-1] 2024-11-18T19:07:03,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T19:07:03,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-18T19:07:03,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788a178a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-18T19:07:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T19:07:03,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 818bff808382,35649,1731956696665 2024-11-18T19:07:03,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@539d434 2024-11-18T19:07:03,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T19:07:03,837 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-18T19:07:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T19:07:03,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=818bff808382,33643,1731956698538, seqNum=-1] 2024-11-18T19:07:03,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T19:07:03,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-18T19:07:03,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-18T19:07:03,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-18T19:07:03,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:07:03,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643. 2024-11-18T19:07:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:07:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:07:03,853 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:07:03,853 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:07:03,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:03,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:07:03,854 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-18T19:07:03,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937., hostname=818bff808382,33643,1731956698538, seqNum=2 , the old value is region=ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937., hostname=818bff808382,33643,1731956698538, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:07:03,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937., hostname=818bff808382,33643,1731956698538, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-18T19:07:03,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937., hostname=818bff808382,33643,1731956698538, seqNum=2 from cache 2024-11-18T19:07:03,854 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-18T19:07:03,871 INFO [Time-limited test {}] impl.RestoreTablesClient(175): No incremental changes since full backup for 'ns2:test-17319567061111', skipping incremental restore step. 2024-11-18T19:07:03,871 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-18T19:07:03,871 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1731956785190 hdfs://localhost:45363/backupUT/backup_1731956785190/ns2/test-17319567061111/ 2024-11-18T19:07:03,871 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-11-18T19:07:03,872 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(217): After incremental restore: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T19:07:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.CallRunner(138): callId: 741 service: ClientService methodName: Scan size: 160 connection: 172.17.0.2:46036 deadline: 1731956883872, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. is not online on 818bff808382,33643,1731956698538 2024-11-18T19:07:03,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2 , the old value is region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. is not online on 818bff808382,33643,1731956698538 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:07:03,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826. is not online on 818bff808382,33643,1731956698538 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:07:03,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1731956798095.2a5f6dfde7b5aa4436340824f71ed826., hostname=818bff808382,33643,1731956698538, seqNum=2 from cache 2024-11-18T19:07:03,962 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=818bff808382:33643 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 34 more 2024-11-18T19:07:03,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., hostname=818bff808382,33643,1731956698538, seqNum=7] 2024-11-18T19:07:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.CallRunner(138): callId: 745 service: ClientService methodName: Scan size: 170 connection: 172.17.0.2:46036 deadline: 1731956883987, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. is not online on 818bff808382,33643,1731956698538 2024-11-18T19:07:03,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce., hostname=818bff808382,33643,1731956698538, seqNum=2 , the old value is region=table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce., hostname=818bff808382,33643,1731956698538, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. is not online on 818bff808382,33643,1731956698538 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:07:03,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce., hostname=818bff808382,33643,1731956698538, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce. is not online on 818bff808382,33643,1731956698538 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:07:03,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,row0,1731956798095.3456776f3aeb3dfd204e897a25e061ce., hostname=818bff808382,33643,1731956698538, seqNum=2 from cache 2024-11-18T19:07:04,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c., hostname=818bff808382,33643,1731956698538, seqNum=7] 2024-11-18T19:07:04,097 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(219): f1 has 198 rows 2024-11-18T19:07:04,100 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(223): f2 has 7 rows 2024-11-18T19:07:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:07:04,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-18T19:07:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:07:04,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-18T19:07:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:07:04,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-18T19:07:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:07:04,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-18T19:07:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:07:04,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-18T19:07:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T19:07:04,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-18T19:07:04,121 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(227): mob has 111 rows 2024-11-18T19:07:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] ipc.CallRunner(138): callId: 753 service: ClientService methodName: Scan size: 159 connection: 172.17.0.2:46036 deadline: 1731956884120, exception=org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. is not online on 818bff808382,33643,1731956698538 2024-11-18T19:07:04,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2 , the old value is region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. is not online on 818bff808382,33643,1731956698538 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:07:04,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603. is not online on 818bff808382,33643,1731956698538 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T19:07:04,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603., hostname=818bff808382,33643,1731956698538, seqNum=2 from cache 2024-11-18T19:07:04,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937., hostname=818bff808382,33643,1731956698538, seqNum=2] 2024-11-18T19:07:04,238 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:07:04,238 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:235) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:07:04,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:04,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:04,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:07:04,253 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=634 (was 611) Potentially hanging thread: member: '818bff808382,33643,1731956698538' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(818bff808382,33643,1731956698538)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-65114922_20 at /127.0.0.1:42012 [Receiving block BP-400906797-172.17.0.2-1731956693525:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/818bff808382:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-400906797-172.17.0.2-1731956693525:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(818bff808382,33643,1731956698538)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/818bff808382:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-400906797-172.17.0.2-1731956693525:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-400906797-172.17.0.2-1731956693525:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-65114922_20 at /127.0.0.1:42026 [Receiving block BP-400906797-172.17.0.2-1731956693525:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-65114922_20 at /127.0.0.1:41998 [Receiving block BP-400906797-172.17.0.2-1731956693525:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/818bff808382:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(818bff808382,33643,1731956698538)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(818bff808382,33643,1731956698538)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (818bff808382,35649,1731956696665)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1886497370_20 at /127.0.0.1:58906 [Waiting for operation #53] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=643 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=108 (was 253), ProcessCount=9 (was 9), AvailableMemoryMB=9168 (was 10962) 2024-11-18T19:07:04,253 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=634 is superior to 500 2024-11-18T19:07:04,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T19:07:04,260 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:07:04,260 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:07:04,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:04,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:04,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:07:04,261 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T19:07:04,261 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=321698493, stopped=false 2024-11-18T19:07:04,261 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=818bff808382,35649,1731956696665 2024-11-18T19:07:04,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-18T19:07:04,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-18T19:07:04,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:07:04,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:07:04,263 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:07:04,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-18T19:07:04,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-18T19:07:04,264 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T19:07:04,265 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:07:04,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:04,265 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '818bff808382,33643,1731956698538' ***** 2024-11-18T19:07:04,265 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T19:07:04,265 INFO [RS:0;818bff808382:33643 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-18T19:07:04,271 INFO [RS:0;818bff808382:33643 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@2b61b876{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-18T19:07:04,273 INFO [RS:0;818bff808382:33643 {}] server.AbstractConnector(383): Stopped ServerConnector@25aa050e{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-18T19:07:04,274 INFO [RS:0;818bff808382:33643 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T19:07:04,274 INFO [RS:0;818bff808382:33643 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@793157ae{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-18T19:07:04,274 INFO [RS:0;818bff808382:33643 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1cdf8417{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,STOPPED} 2024-11-18T19:07:04,276 INFO [RS:0;818bff808382:33643 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T19:07:04,276 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T19:07:04,276 INFO [RS:0;818bff808382:33643 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T19:07:04,276 INFO [RS:0;818bff808382:33643 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for 1c7d96a664e933b6430bd1488e8a2937 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for e689fbc273d3ab005a435f927e214efb 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for b38dbac143e9407aa48a95e23085ace3 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for d144bc0c998ecf0ed9ce62cb631ee3eb 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for 63e71206ef16a51a40e751c8e3988243 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for cf5451ece9935ba018e98673baf4e95b 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for cd0db3f2ace15f2ccb5d44bb7a079f9c 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for f44dca6ed6ddff070a4eead3e92d66bc 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for f88044664c926e5a290147e827b7cf95 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(959): stopping server 818bff808382,33643,1731956698538 2024-11-18T19:07:04,277 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1c7d96a664e933b6430bd1488e8a2937, disabling compactions & flushes 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T19:07:04,277 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:07:04,277 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:07:04,277 INFO [RS:0;818bff808382:33643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;818bff808382:33643. 2024-11-18T19:07:04,277 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. after waiting 0 ms 2024-11-18T19:07:04,277 INFO [regionserver/818bff808382:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T19:07:04,277 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:07:04,277 DEBUG [RS:0;818bff808382:33643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:07:04,278 DEBUG [RS:0;818bff808382:33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:04,278 INFO [RS:0;818bff808382:33643 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T19:07:04,278 INFO [RS:0;818bff808382:33643 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T19:07:04,278 INFO [RS:0;818bff808382:33643 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T19:07:04,278 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T19:07:04,278 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-18T19:07:04,278 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T19:07:04,278 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T19:07:04,278 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T19:07:04,278 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T19:07:04,278 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T19:07:04,279 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1c7d96a664e933b6430bd1488e8a2937, 63e71206ef16a51a40e751c8e3988243, b38dbac143e9407aa48a95e23085ace3, cd0db3f2ace15f2ccb5d44bb7a079f9c, cf5451ece9935ba018e98673baf4e95b, d144bc0c998ecf0ed9ce62cb631ee3eb, e689fbc273d3ab005a435f927e214efb, f44dca6ed6ddff070a4eead3e92d66bc, f88044664c926e5a290147e827b7cf95 2024-11-18T19:07:04,279 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=23.97 KB heapSize=42.40 KB 2024-11-18T19:07:04,282 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/table2/1c7d96a664e933b6430bd1488e8a2937/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-18T19:07:04,283 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:04,283 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:07:04,283 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1c7d96a664e933b6430bd1488e8a2937: Waiting for close lock at 1731956824277Running coprocessor pre-close hooks at 1731956824277Disabling compacts and flushes for region at 1731956824277Disabling writes for close at 1731956824277Writing region close event to WAL at 1731956824278 (+1 ms)Running coprocessor post-close hooks at 1731956824282 (+4 ms)Closed at 1731956824283 (+1 ms) 2024-11-18T19:07:04,283 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:table2,,1731956819068.1c7d96a664e933b6430bd1488e8a2937. 2024-11-18T19:07:04,283 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e689fbc273d3ab005a435f927e214efb, disabling compactions & flushes 2024-11-18T19:07:04,283 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:07:04,283 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:07:04,283 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. after waiting 0 ms 2024-11-18T19:07:04,283 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:07:04,286 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system_bulk/e689fbc273d3ab005a435f927e214efb/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-18T19:07:04,287 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:04,287 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:07:04,287 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e689fbc273d3ab005a435f927e214efb: Waiting for close lock at 1731956824283Running coprocessor pre-close hooks at 1731956824283Disabling compacts and flushes for region at 1731956824283Disabling writes for close at 1731956824283Writing region close event to WAL at 1731956824283Running coprocessor post-close hooks at 1731956824287 (+4 ms)Closed at 1731956824287 2024-11-18T19:07:04,287 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1731956716572.e689fbc273d3ab005a435f927e214efb. 2024-11-18T19:07:04,287 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b38dbac143e9407aa48a95e23085ace3, disabling compactions & flushes 2024-11-18T19:07:04,287 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:07:04,287 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:07:04,287 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. after waiting 0 ms 2024-11-18T19:07:04,287 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:07:04,287 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b38dbac143e9407aa48a95e23085ace3 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-18T19:07:04,302 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/info/ad9d421ee7ab47b8992dd1b92027ba45 is 159, key is ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb./info:regioninfo/1731956709319/Put/seqid=0 2024-11-18T19:07:04,302 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f2/dabce52db66b4519b85e31acdf54f659 is 58, key is row-test-1731956706111-2-0/f2:q1/1731956780182/Put/seqid=0 2024-11-18T19:07:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741997_1173 (size=19093) 2024-11-18T19:07:04,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741996_1172 (size=5464) 2024-11-18T19:07:04,479 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 63e71206ef16a51a40e751c8e3988243, b38dbac143e9407aa48a95e23085ace3, cd0db3f2ace15f2ccb5d44bb7a079f9c, cf5451ece9935ba018e98673baf4e95b, d144bc0c998ecf0ed9ce62cb631ee3eb, f44dca6ed6ddff070a4eead3e92d66bc, f88044664c926e5a290147e827b7cf95 2024-11-18T19:07:04,679 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 63e71206ef16a51a40e751c8e3988243, b38dbac143e9407aa48a95e23085ace3, cd0db3f2ace15f2ccb5d44bb7a079f9c, cf5451ece9935ba018e98673baf4e95b, d144bc0c998ecf0ed9ce62cb631ee3eb, f44dca6ed6ddff070a4eead3e92d66bc, f88044664c926e5a290147e827b7cf95 2024-11-18T19:07:04,707 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f2/dabce52db66b4519b85e31acdf54f659 2024-11-18T19:07:04,707 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.86 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/info/ad9d421ee7ab47b8992dd1b92027ba45 2024-11-18T19:07:04,714 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/.tmp/f2/dabce52db66b4519b85e31acdf54f659 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f2/dabce52db66b4519b85e31acdf54f659 2024-11-18T19:07:04,719 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/f2/dabce52db66b4519b85e31acdf54f659, entries=7, sequenceid=348, filesize=5.3 K 2024-11-18T19:07:04,720 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for b38dbac143e9407aa48a95e23085ace3 in 433ms, sequenceid=348, compaction requested=false 2024-11-18T19:07:04,728 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/ns/bbb3efb734a4406e9a6f0029c96b0cb8 is 85, key is ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603./ns:/1731956819068/DeleteFamily/seqid=0 2024-11-18T19:07:04,730 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/test-1731956706111/b38dbac143e9407aa48a95e23085ace3/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-18T19:07:04,731 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:04,731 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:07:04,731 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b38dbac143e9407aa48a95e23085ace3: Waiting for close lock at 1731956824287Running coprocessor pre-close hooks at 1731956824287Disabling compacts and flushes for region at 1731956824287Disabling writes for close at 1731956824287Obtaining lock to block concurrent updates at 1731956824287Preparing flush snapshotting stores in b38dbac143e9407aa48a95e23085ace3 at 1731956824287Finished memstore snapshotting test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1731956824288 (+1 ms)Flushing stores of test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. at 1731956824288Flushing b38dbac143e9407aa48a95e23085ace3/f2: creating writer at 1731956824288Flushing b38dbac143e9407aa48a95e23085ace3/f2: appending metadata at 1731956824302 (+14 ms)Flushing b38dbac143e9407aa48a95e23085ace3/f2: closing flushed file at 1731956824302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70e38c53: reopening flushed file at 1731956824713 (+411 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for b38dbac143e9407aa48a95e23085ace3 in 433ms, sequenceid=348, compaction requested=false at 1731956824720 (+7 ms)Writing region close event to WAL at 1731956824727 (+7 ms)Running coprocessor post-close hooks at 1731956824731 (+4 ms)Closed at 1731956824731 2024-11-18T19:07:04,731 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1731956706111,,1731956706608.b38dbac143e9407aa48a95e23085ace3. 2024-11-18T19:07:04,731 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d144bc0c998ecf0ed9ce62cb631ee3eb, disabling compactions & flushes 2024-11-18T19:07:04,731 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:07:04,731 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:07:04,731 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. after waiting 0 ms 2024-11-18T19:07:04,732 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:07:04,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741998_1174 (size=5760) 2024-11-18T19:07:04,735 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns2/test-17319567061111/d144bc0c998ecf0ed9ce62cb631ee3eb/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-18T19:07:04,736 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:04,736 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:07:04,736 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d144bc0c998ecf0ed9ce62cb631ee3eb: Waiting for close lock at 1731956824731Running coprocessor pre-close hooks at 1731956824731Disabling compacts and flushes for region at 1731956824731Disabling writes for close at 1731956824731Writing region close event to WAL at 1731956824732 (+1 ms)Running coprocessor post-close hooks at 1731956824736 (+4 ms)Closed at 1731956824736 2024-11-18T19:07:04,736 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17319567061111,,1731956708939.d144bc0c998ecf0ed9ce62cb631ee3eb. 2024-11-18T19:07:04,736 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 63e71206ef16a51a40e751c8e3988243, disabling compactions & flushes 2024-11-18T19:07:04,736 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:07:04,736 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:07:04,736 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. after waiting 0 ms 2024-11-18T19:07:04,736 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:07:04,736 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 63e71206ef16a51a40e751c8e3988243 2/2 column families, dataSize=2.12 KB heapSize=3.43 KB 2024-11-18T19:07:04,740 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4dff235cb8214ae4ac7617ed7153eff3 is 158, key is trslm:hdfs://localhost:45363/backupUT\x00ns2:test-17319567061111/meta:log-roll-map/1731956797610/Put/seqid=0 2024-11-18T19:07:04,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741999_1175 (size=6208) 2024-11-18T19:07:04,879 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 63e71206ef16a51a40e751c8e3988243, cd0db3f2ace15f2ccb5d44bb7a079f9c, cf5451ece9935ba018e98673baf4e95b, f44dca6ed6ddff070a4eead3e92d66bc, f88044664c926e5a290147e827b7cf95 2024-11-18T19:07:05,079 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 63e71206ef16a51a40e751c8e3988243, cd0db3f2ace15f2ccb5d44bb7a079f9c, cf5451ece9935ba018e98673baf4e95b, f44dca6ed6ddff070a4eead3e92d66bc, f88044664c926e5a290147e827b7cf95 2024-11-18T19:07:05,132 INFO [regionserver/818bff808382:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T19:07:05,132 INFO [regionserver/818bff808382:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T19:07:05,133 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=474 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/ns/bbb3efb734a4406e9a6f0029c96b0cb8 2024-11-18T19:07:05,144 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=944 B at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4dff235cb8214ae4ac7617ed7153eff3 2024-11-18T19:07:05,151 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/ca1e6703d09145a9b69ea8b2fe13a263 is 588, key is session:backup_1731956785190/session:context/1731956798040/Put/seqid=0 2024-11-18T19:07:05,155 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/rep_barrier/48e66f6af4454e69811e427bebae2c6e is 94, key is ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603./rep_barrier:/1731956819068/DeleteFamily/seqid=0 2024-11-18T19:07:05,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073742000_1176 (size=5689) 2024-11-18T19:07:05,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073742001_1177 (size=5640) 2024-11-18T19:07:05,280 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-18T19:07:05,280 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 63e71206ef16a51a40e751c8e3988243=backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., cf5451ece9935ba018e98673baf4e95b=table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b., cd0db3f2ace15f2ccb5d44bb7a079f9c=table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c., f44dca6ed6ddff070a4eead3e92d66bc=ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc., f88044664c926e5a290147e827b7cf95=ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95.} 2024-11-18T19:07:05,280 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 63e71206ef16a51a40e751c8e3988243, cd0db3f2ace15f2ccb5d44bb7a079f9c, cf5451ece9935ba018e98673baf4e95b, f44dca6ed6ddff070a4eead3e92d66bc, f88044664c926e5a290147e827b7cf95 2024-11-18T19:07:05,480 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 63e71206ef16a51a40e751c8e3988243, cd0db3f2ace15f2ccb5d44bb7a079f9c, cf5451ece9935ba018e98673baf4e95b, f44dca6ed6ddff070a4eead3e92d66bc, f88044664c926e5a290147e827b7cf95 2024-11-18T19:07:05,556 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/ca1e6703d09145a9b69ea8b2fe13a263 2024-11-18T19:07:05,559 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=266 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/rep_barrier/48e66f6af4454e69811e427bebae2c6e 2024-11-18T19:07:05,562 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/meta/4dff235cb8214ae4ac7617ed7153eff3 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4dff235cb8214ae4ac7617ed7153eff3 2024-11-18T19:07:05,567 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/meta/4dff235cb8214ae4ac7617ed7153eff3, entries=7, sequenceid=52, filesize=6.1 K 2024-11-18T19:07:05,568 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/.tmp/session/ca1e6703d09145a9b69ea8b2fe13a263 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/ca1e6703d09145a9b69ea8b2fe13a263 2024-11-18T19:07:05,572 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/session/ca1e6703d09145a9b69ea8b2fe13a263, entries=2, sequenceid=52, filesize=5.6 K 2024-11-18T19:07:05,573 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for 63e71206ef16a51a40e751c8e3988243 in 837ms, sequenceid=52, compaction requested=true 2024-11-18T19:07:05,578 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/backup/system/63e71206ef16a51a40e751c8e3988243/recovered.edits/55.seqid, newMaxSeqId=55, maxSeqId=1 2024-11-18T19:07:05,578 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:05,578 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:07:05,578 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 63e71206ef16a51a40e751c8e3988243: Waiting for close lock at 1731956824736Running coprocessor pre-close hooks at 1731956824736Disabling compacts and flushes for region at 1731956824736Disabling writes for close at 1731956824736Obtaining lock to block concurrent updates at 1731956824736Preparing flush snapshotting stores in 63e71206ef16a51a40e751c8e3988243 at 1731956824736Finished memstore snapshotting backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243., syncing WAL and waiting on mvcc, flushsize=dataSize=2170, getHeapSize=3480, getOffHeapSize=0, getCellsCount=11 at 1731956824737 (+1 ms)Flushing stores of backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. at 1731956824737Flushing 63e71206ef16a51a40e751c8e3988243/meta: creating writer at 1731956824737Flushing 63e71206ef16a51a40e751c8e3988243/meta: appending metadata at 1731956824739 (+2 ms)Flushing 63e71206ef16a51a40e751c8e3988243/meta: closing flushed file at 1731956824739Flushing 63e71206ef16a51a40e751c8e3988243/session: creating writer at 1731956825148 (+409 ms)Flushing 63e71206ef16a51a40e751c8e3988243/session: appending metadata at 1731956825151 (+3 ms)Flushing 63e71206ef16a51a40e751c8e3988243/session: closing flushed file at 1731956825151Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d3337b3: reopening flushed file at 1731956825561 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41db5dd: reopening flushed file at 1731956825567 (+6 ms)Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for 63e71206ef16a51a40e751c8e3988243 in 837ms, sequenceid=52, compaction requested=true at 1731956825573 (+6 ms)Writing region close event to WAL at 1731956825575 (+2 ms)Running coprocessor post-close hooks at 1731956825578 (+3 ms)Closed at 1731956825578 2024-11-18T19:07:05,579 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1731956714400.63e71206ef16a51a40e751c8e3988243. 2024-11-18T19:07:05,579 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf5451ece9935ba018e98673baf4e95b, disabling compactions & flushes 2024-11-18T19:07:05,579 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:07:05,579 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:07:05,579 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. after waiting 0 ms 2024-11-18T19:07:05,579 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:07:05,582 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/table/f078da1a20804730a85993e0ea921ca7 is 88, key is ns2:table2,,1731956804029.7cca13c6ec211bff31d3fc7f72705603./table:/1731956819068/DeleteFamily/seqid=0 2024-11-18T19:07:05,583 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cf5451ece9935ba018e98673baf4e95b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=6 2024-11-18T19:07:05,583 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:05,583 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:07:05,583 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf5451ece9935ba018e98673baf4e95b: Waiting for close lock at 1731956825579Running coprocessor pre-close hooks at 1731956825579Disabling compacts and flushes for region at 1731956825579Disabling writes for close at 1731956825579Writing region close event to WAL at 1731956825580 (+1 ms)Running coprocessor post-close hooks at 1731956825583 (+3 ms)Closed at 1731956825583 2024-11-18T19:07:05,583 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731956809237.cf5451ece9935ba018e98673baf4e95b. 2024-11-18T19:07:05,584 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd0db3f2ace15f2ccb5d44bb7a079f9c, disabling compactions & flushes 2024-11-18T19:07:05,584 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:07:05,584 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:07:05,584 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. after waiting 0 ms 2024-11-18T19:07:05,584 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:07:05,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073742002_1178 (size=6364) 2024-11-18T19:07:05,587 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/default/table1/cd0db3f2ace15f2ccb5d44bb7a079f9c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=6 2024-11-18T19:07:05,588 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:05,588 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:07:05,588 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd0db3f2ace15f2ccb5d44bb7a079f9c: Waiting for close lock at 1731956825584Running coprocessor pre-close hooks at 1731956825584Disabling compacts and flushes for region at 1731956825584Disabling writes for close at 1731956825584Writing region close event to WAL at 1731956825584Running coprocessor post-close hooks at 1731956825588 (+4 ms)Closed at 1731956825588 2024-11-18T19:07:05,588 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,row0,1731956809237.cd0db3f2ace15f2ccb5d44bb7a079f9c. 2024-11-18T19:07:05,588 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f44dca6ed6ddff070a4eead3e92d66bc, disabling compactions & flushes 2024-11-18T19:07:05,588 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:07:05,588 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:07:05,588 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. after waiting 0 ms 2024-11-18T19:07:05,588 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:07:05,590 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns3/test-17319567061112/f44dca6ed6ddff070a4eead3e92d66bc/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-18T19:07:05,591 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:05,591 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:07:05,591 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f44dca6ed6ddff070a4eead3e92d66bc: Waiting for close lock at 1731956825588Running coprocessor pre-close hooks at 1731956825588Disabling compacts and flushes for region at 1731956825588Disabling writes for close at 1731956825588Writing region close event to WAL at 1731956825588Running coprocessor post-close hooks at 1731956825591 (+3 ms)Closed at 1731956825591 2024-11-18T19:07:05,591 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17319567061112,,1731956709713.f44dca6ed6ddff070a4eead3e92d66bc. 2024-11-18T19:07:05,591 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f88044664c926e5a290147e827b7cf95, disabling compactions & flushes 2024-11-18T19:07:05,591 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:07:05,591 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:07:05,591 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. after waiting 0 ms 2024-11-18T19:07:05,591 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:07:05,594 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/ns4/test-17319567061113/f88044664c926e5a290147e827b7cf95/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-18T19:07:05,594 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:05,594 INFO [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:07:05,594 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f88044664c926e5a290147e827b7cf95: Waiting for close lock at 1731956825591Running coprocessor pre-close hooks at 1731956825591Disabling compacts and flushes for region at 1731956825591Disabling writes for close at 1731956825591Writing region close event to WAL at 1731956825592 (+1 ms)Running coprocessor post-close hooks at 1731956825594 (+2 ms)Closed at 1731956825594 2024-11-18T19:07:05,594 DEBUG [RS_CLOSE_REGION-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17319567061113,,1731956710354.f88044664c926e5a290147e827b7cf95. 2024-11-18T19:07:05,680 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T19:07:05,880 DEBUG [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T19:07:05,986 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.39 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/table/f078da1a20804730a85993e0ea921ca7 2024-11-18T19:07:05,992 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/info/ad9d421ee7ab47b8992dd1b92027ba45 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/info/ad9d421ee7ab47b8992dd1b92027ba45 2024-11-18T19:07:05,997 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/info/ad9d421ee7ab47b8992dd1b92027ba45, entries=114, sequenceid=97, filesize=18.6 K 2024-11-18T19:07:05,998 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/ns/bbb3efb734a4406e9a6f0029c96b0cb8 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/ns/bbb3efb734a4406e9a6f0029c96b0cb8 2024-11-18T19:07:06,002 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/ns/bbb3efb734a4406e9a6f0029c96b0cb8, entries=10, sequenceid=97, filesize=5.6 K 2024-11-18T19:07:06,003 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/rep_barrier/48e66f6af4454e69811e427bebae2c6e as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/rep_barrier/48e66f6af4454e69811e427bebae2c6e 2024-11-18T19:07:06,007 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/rep_barrier/48e66f6af4454e69811e427bebae2c6e, entries=3, sequenceid=97, filesize=5.5 K 2024-11-18T19:07:06,008 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/.tmp/table/f078da1a20804730a85993e0ea921ca7 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/table/f078da1a20804730a85993e0ea921ca7 2024-11-18T19:07:06,012 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/table/f078da1a20804730a85993e0ea921ca7, entries=21, sequenceid=97, filesize=6.2 K 2024-11-18T19:07:06,013 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1735ms, sequenceid=97, compaction requested=false 2024-11-18T19:07:06,017 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/data/hbase/meta/1588230740/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2024-11-18T19:07:06,018 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:06,018 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T19:07:06,018 INFO [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T19:07:06,018 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731956824278Running coprocessor pre-close hooks at 1731956824278Disabling compacts and flushes for region at 1731956824278Disabling writes for close at 1731956824278Obtaining lock to block concurrent updates at 1731956824279 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731956824279Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=24544, getHeapSize=43352, getOffHeapSize=0, getCellsCount=235 at 1731956824279Flushing stores of hbase:meta,,1.1588230740 at 1731956824279Flushing 1588230740/info: creating writer at 1731956824279Flushing 1588230740/info: appending metadata at 1731956824302 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731956824302Flushing 1588230740/ns: creating writer at 1731956824713 (+411 ms)Flushing 1588230740/ns: appending metadata at 1731956824728 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731956824728Flushing 1588230740/rep_barrier: creating writer at 1731956825138 (+410 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731956825154 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731956825154Flushing 1588230740/table: creating writer at 1731956825563 (+409 ms)Flushing 1588230740/table: appending metadata at 1731956825581 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731956825581Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2818ca14: reopening flushed file at 1731956825991 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c6f5a66: reopening flushed file at 1731956825997 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35e1a13f: reopening flushed file at 1731956826002 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42509092: reopening flushed file at 1731956826007 (+5 ms)Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1735ms, sequenceid=97, compaction requested=false at 1731956826013 (+6 ms)Writing region close event to WAL at 1731956826014 (+1 ms)Running coprocessor post-close hooks at 1731956826017 (+3 ms)Closed at 1731956826018 (+1 ms) 2024-11-18T19:07:06,018 DEBUG [RS_CLOSE_META-regionserver/818bff808382:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T19:07:06,081 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(976): stopping server 818bff808382,33643,1731956698538; all regions closed. 2024-11-18T19:07:06,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741955_1131 (size=16160) 2024-11-18T19:07:06,084 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.meta.1731956789560.meta not finished, retry = 0 2024-11-18T19:07:06,188 DEBUG [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs 2024-11-18T19:07:06,188 INFO [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 818bff808382%2C33643%2C1731956698538.meta:.meta(num 1731956789560) 2024-11-18T19:07:06,190 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0.1731956789578 not finished, retry = 0 2024-11-18T19:07:06,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741956_1132 (size=4491) 2024-11-18T19:07:06,293 DEBUG [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs 2024-11-18T19:07:06,293 INFO [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-0:(num 1731956789578) 2024-11-18T19:07:06,295 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/WALs/818bff808382,33643,1731956698538/818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1.1731956789542 not finished, retry = 0 2024-11-18T19:07:06,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741954_1130 (size=7557) 2024-11-18T19:07:06,399 DEBUG [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/oldWALs 2024-11-18T19:07:06,399 INFO [RS:0;818bff808382:33643 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 818bff808382%2C33643%2C1731956698538.818bff808382%2C33643%2C1731956698538.regiongroup-1:(num 1731956789542) 2024-11-18T19:07:06,399 DEBUG [RS:0;818bff808382:33643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:06,399 INFO [RS:0;818bff808382:33643 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T19:07:06,399 INFO [RS:0;818bff808382:33643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T19:07:06,400 INFO [RS:0;818bff808382:33643 {}] hbase.ChoreService(370): Chore service for: regionserver/818bff808382:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T19:07:06,400 INFO [RS:0;818bff808382:33643 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T19:07:06,400 INFO [regionserver/818bff808382:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T19:07:06,401 INFO [RS:0;818bff808382:33643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33643 2024-11-18T19:07:06,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/818bff808382,33643,1731956698538 2024-11-18T19:07:06,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-18T19:07:06,405 INFO [RS:0;818bff808382:33643 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T19:07:06,407 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [818bff808382,33643,1731956698538] 2024-11-18T19:07:06,409 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/818bff808382,33643,1731956698538 already deleted, retry=false 2024-11-18T19:07:06,409 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 818bff808382,33643,1731956698538 expired; onlineServers=0 2024-11-18T19:07:06,409 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '818bff808382,35649,1731956696665' ***** 2024-11-18T19:07:06,409 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T19:07:06,409 INFO [M:0;818bff808382:35649 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-18T19:07:06,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-18T19:07:06,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-18T19:07:06,415 INFO [M:0;818bff808382:35649 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@78825dd4{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-18T19:07:06,415 INFO [M:0;818bff808382:35649 {}] server.AbstractConnector(383): Stopped ServerConnector@7952237b{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-18T19:07:06,415 INFO [M:0;818bff808382:35649 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T19:07:06,416 INFO [M:0;818bff808382:35649 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@51ab1ff8{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-18T19:07:06,416 INFO [M:0;818bff808382:35649 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@bc9258d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,STOPPED} 2024-11-18T19:07:06,416 INFO [M:0;818bff808382:35649 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T19:07:06,416 INFO [M:0;818bff808382:35649 {}] master.HMaster(1770): Stopping master jetty server 2024-11-18T19:07:06,417 INFO [M:0;818bff808382:35649 {}] server.AbstractConnector(383): Stopped ServerConnector@4b1b14e2{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-18T19:07:06,417 INFO [M:0;818bff808382:35649 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T19:07:06,417 DEBUG [M:0;818bff808382:35649 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T19:07:06,418 INFO [M:0;818bff808382:35649 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-18T19:07:06,418 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T19:07:06,418 INFO [M:0;818bff808382:35649 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;818bff808382:35649. 2024-11-18T19:07:06,418 DEBUG [master/818bff808382:0:becomeActiveMaster-HFileCleaner.large.0-1731956700900 {}] cleaner.HFileCleaner(306): Exit Thread[master/818bff808382:0:becomeActiveMaster-HFileCleaner.large.0-1731956700900,5,FailOnTimeoutGroup] 2024-11-18T19:07:06,418 DEBUG [master/818bff808382:0:becomeActiveMaster-HFileCleaner.small.0-1731956700901 {}] cleaner.HFileCleaner(306): Exit Thread[master/818bff808382:0:becomeActiveMaster-HFileCleaner.small.0-1731956700901,5,FailOnTimeoutGroup] 2024-11-18T19:07:06,418 DEBUG [M:0;818bff808382:35649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T19:07:06,418 DEBUG [M:0;818bff808382:35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:06,418 DEBUG [M:0;818bff808382:35649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T19:07:06,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T19:07:06,418 DEBUG [M:0;818bff808382:35649 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T19:07:06,418 INFO [M:0;818bff808382:35649 {}] hbase.ChoreService(370): Chore service for: master/818bff808382:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T19:07:06,419 INFO [M:0;818bff808382:35649 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T19:07:06,419 DEBUG [M:0;818bff808382:35649 {}] master.HMaster(1795): Stopping service threads 2024-11-18T19:07:06,419 INFO [M:0;818bff808382:35649 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T19:07:06,419 INFO [M:0;818bff808382:35649 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T19:07:06,419 ERROR [M:0;818bff808382:35649 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-1,5,PEWorkerGroup] Thread[HFileArchiver-2,5,PEWorkerGroup] Thread[HFileArchiver-3,5,PEWorkerGroup] 2024-11-18T19:07:06,420 INFO [M:0;818bff808382:35649 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T19:07:06,420 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T19:07:06,420 DEBUG [M:0;818bff808382:35649 {}] zookeeper.ZKUtil(347): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-18T19:07:06,420 WARN [M:0;818bff808382:35649 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T19:07:06,421 INFO [M:0;818bff808382:35649 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/.lastflushedseqids 2024-11-18T19:07:06,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073742003_1179 (size=761) 2024-11-18T19:07:06,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T19:07:06,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33643-0x1000c059bf10001, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T19:07:06,508 INFO [RS:0;818bff808382:33643 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T19:07:06,509 INFO [RS:0;818bff808382:33643 {}] regionserver.HRegionServer(1031): Exiting; stopping=818bff808382,33643,1731956698538; zookeeper connection closed. 2024-11-18T19:07:06,509 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7f0170d4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7f0170d4 2024-11-18T19:07:06,509 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T19:07:06,833 INFO [M:0;818bff808382:35649 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T19:07:06,833 INFO [M:0;818bff808382:35649 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T19:07:06,833 DEBUG [M:0;818bff808382:35649 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T19:07:06,847 INFO [M:0;818bff808382:35649 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:07:06,847 DEBUG [M:0;818bff808382:35649 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:07:06,847 DEBUG [M:0;818bff808382:35649 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T19:07:06,847 DEBUG [M:0;818bff808382:35649 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:07:06,847 INFO [M:0;818bff808382:35649 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=441.39 KB heapSize=518.41 KB 2024-11-18T19:07:06,863 DEBUG [M:0;818bff808382:35649 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55ff3f5dba9049bbaeac95bd94e3121f is 82, key is hbase:meta,,1/info:regioninfo/1731956701779/Put/seqid=0 2024-11-18T19:07:06,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073742004_1180 (size=5672) 2024-11-18T19:07:07,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T19:07:07,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17319567061112 2024-11-18T19:07:07,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-18T19:07:07,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-11-18T19:07:07,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T19:07:07,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1731956706111 2024-11-18T19:07:07,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-18T19:07:07,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17319567061111 2024-11-18T19:07:07,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17319567061113 2024-11-18T19:07:07,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-18T19:07:07,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-11-18T19:07:07,268 INFO [M:0;818bff808382:35649 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55ff3f5dba9049bbaeac95bd94e3121f 2024-11-18T19:07:07,290 DEBUG [M:0;818bff808382:35649 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/129e6483db9f41eb8ccae54ed47812d1 is 2201, key is \x00\x00\x00\x00\x00\x00\x00Q/proc:d/1731956816063/Put/seqid=0 2024-11-18T19:07:07,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073742005_1181 (size=30165) 2024-11-18T19:07:07,695 INFO [M:0;818bff808382:35649 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=440.84 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/129e6483db9f41eb8ccae54ed47812d1 2024-11-18T19:07:07,700 INFO [M:0;818bff808382:35649 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 129e6483db9f41eb8ccae54ed47812d1 2024-11-18T19:07:07,715 DEBUG [M:0;818bff808382:35649 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/520068e345af4694b093c70b2197df46 is 69, key is 818bff808382,33643,1731956698538/rs:state/1731956700995/Put/seqid=0 2024-11-18T19:07:07,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073742006_1182 (size=5156) 2024-11-18T19:07:08,121 INFO [M:0;818bff808382:35649 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/520068e345af4694b093c70b2197df46 2024-11-18T19:07:08,127 DEBUG [M:0;818bff808382:35649 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55ff3f5dba9049bbaeac95bd94e3121f as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/55ff3f5dba9049bbaeac95bd94e3121f 2024-11-18T19:07:08,131 INFO [M:0;818bff808382:35649 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/55ff3f5dba9049bbaeac95bd94e3121f, entries=8, sequenceid=908, filesize=5.5 K 2024-11-18T19:07:08,132 DEBUG [M:0;818bff808382:35649 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/129e6483db9f41eb8ccae54ed47812d1 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/129e6483db9f41eb8ccae54ed47812d1 2024-11-18T19:07:08,136 INFO [M:0;818bff808382:35649 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 129e6483db9f41eb8ccae54ed47812d1 2024-11-18T19:07:08,136 INFO [M:0;818bff808382:35649 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/129e6483db9f41eb8ccae54ed47812d1, entries=94, sequenceid=908, filesize=29.5 K 2024-11-18T19:07:08,137 DEBUG [M:0;818bff808382:35649 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/520068e345af4694b093c70b2197df46 as hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/520068e345af4694b093c70b2197df46 2024-11-18T19:07:08,141 INFO [M:0;818bff808382:35649 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45363/user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/520068e345af4694b093c70b2197df46, entries=1, sequenceid=908, filesize=5.0 K 2024-11-18T19:07:08,142 INFO [M:0;818bff808382:35649 {}] regionserver.HRegion(3140): Finished flush of dataSize ~441.39 KB/451988, heapSize ~518.12 KB/530552, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1295ms, sequenceid=908, compaction requested=false 2024-11-18T19:07:08,143 INFO [M:0;818bff808382:35649 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T19:07:08,143 DEBUG [M:0;818bff808382:35649 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731956826833Disabling compacts and flushes for region at 1731956826833Disabling writes for close at 1731956826847 (+14 ms)Obtaining lock to block concurrent updates at 1731956826847Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731956826847Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=451988, getHeapSize=530792, getOffHeapSize=0, getCellsCount=1031 at 1731956826848 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731956826848Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731956826848Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731956826863 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731956826863Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731956827273 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731956827289 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731956827289Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731956827700 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731956827714 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731956827714Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7990615d: reopening flushed file at 1731956828126 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6161dc11: reopening flushed file at 1731956828131 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ea743ba: reopening flushed file at 1731956828136 (+5 ms)Finished flush of dataSize ~441.39 KB/451988, heapSize ~518.12 KB/530552, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1295ms, sequenceid=908, compaction requested=false at 1731956828142 (+6 ms)Writing region close event to WAL at 1731956828143 (+1 ms)Closed at 1731956828143 2024-11-18T19:07:08,146 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ca7c40f-db87-8546-e5ac-041c132033e9/MasterData/WALs/818bff808382,35649,1731956696665/818bff808382%2C35649%2C1731956696665.818bff808382%2C35649%2C1731956696665.regiongroup-0.1731956700266 not finished, retry = 0 2024-11-18T19:07:08,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33673 is added to blk_1073741830_1006 (size=511919) 2024-11-18T19:07:08,247 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T19:07:08,247 INFO [M:0;818bff808382:35649 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T19:07:08,247 INFO [M:0;818bff808382:35649 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-18T19:07:08,248 INFO [M:0;818bff808382:35649 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35649 2024-11-18T19:07:08,248 INFO [M:0;818bff808382:35649 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T19:07:08,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T19:07:08,353 INFO [M:0;818bff808382:35649 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T19:07:08,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35649-0x1000c059bf10000, quorum=127.0.0.1:64381, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T19:07:08,363 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@cb44ad4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T19:07:08,365 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T19:07:08,365 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T19:07:08,365 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T19:07:08,366 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,STOPPED} 2024-11-18T19:07:08,368 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T19:07:08,368 WARN [BP-400906797-172.17.0.2-1731956693525 heartbeating to localhost/127.0.0.1:45363 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T19:07:08,368 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T19:07:08,368 WARN [BP-400906797-172.17.0.2-1731956693525 heartbeating to localhost/127.0.0.1:45363 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-400906797-172.17.0.2-1731956693525 (Datanode Uuid e75a332a-38ee-4bff-a5be-9a275960cd0b) service to localhost/127.0.0.1:45363 2024-11-18T19:07:08,370 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/data/data1/current/BP-400906797-172.17.0.2-1731956693525 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T19:07:08,370 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/cluster_e2ab2741-0620-bb64-3a84-4ce49863dd31/data/data2/current/BP-400906797-172.17.0.2-1731956693525 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T19:07:08,371 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T19:07:08,377 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@38e78c7b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T19:07:08,378 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c336369{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T19:07:08,378 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T19:07:08,378 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b22dca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T19:07:08,378 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f045f3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,STOPPED} 2024-11-18T19:07:08,387 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T19:07:08,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T19:07:08,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-18T19:07:08,454 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7843287{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-18T19:07:08,455 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@67510acc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T19:07:08,455 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T19:07:08,455 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@729dadbb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-18T19:07:08,455 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@297e0e95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,STOPPED} 2024-11-18T19:07:08,464 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ceda6b8{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-18T19:07:08,465 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@b86af18{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T19:07:08,465 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T19:07:08,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25e99e8f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-18T19:07:08,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3cf29358{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,STOPPED} 2024-11-18T19:07:08,480 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-18T19:07:08,480 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5dc6a938{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-18T19:07:08,481 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@37a72b0b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T19:07:08,481 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T19:07:08,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ca0b593{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-18T19:07:08,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c57c6a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,STOPPED} 2024-11-18T19:07:08,484 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-18T19:07:08,489 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-18T19:07:08,489 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-18T19:07:08,491 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-18T19:07:08,493 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@355625e7{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-18T19:07:08,494 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@73ba90c4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T19:07:08,494 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T19:07:08,494 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a80b569{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-18T19:07:08,494 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45067f3b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/59967c5f-01f5-3f42-3fad-76a593a45a5c/hadoop.log.dir/,STOPPED} 2024-11-18T19:07:08,496 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-18T19:07:08,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped