2024-11-26 13:35:52,332 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-26 13:35:52,352 main DEBUG Took 0.014993 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 13:35:52,353 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 13:35:52,353 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 13:35:52,354 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 13:35:52,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,380 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 13:35:52,402 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,404 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,405 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,405 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,406 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,406 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,407 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,408 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,410 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,410 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,411 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,412 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,412 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,413 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,413 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,414 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,414 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,415 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,416 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,417 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 13:35:52,418 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,418 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 13:35:52,420 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 13:35:52,422 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 13:35:52,425 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 13:35:52,425 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 13:35:52,427 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 13:35:52,428 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 13:35:52,451 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 13:35:52,456 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 13:35:52,458 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 13:35:52,459 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 13:35:52,459 main DEBUG createAppenders(={Console}) 2024-11-26 13:35:52,460 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-26 13:35:52,461 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-26 13:35:52,461 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-26 13:35:52,462 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 13:35:52,462 main DEBUG OutputStream closed 2024-11-26 13:35:52,463 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 13:35:52,463 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 13:35:52,463 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-26 13:35:52,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 13:35:52,592 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 13:35:52,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 13:35:52,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 13:35:52,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 13:35:52,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 13:35:52,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 13:35:52,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 13:35:52,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 13:35:52,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 13:35:52,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 13:35:52,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 13:35:52,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 13:35:52,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 13:35:52,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 13:35:52,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 13:35:52,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 13:35:52,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 13:35:52,606 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 13:35:52,606 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-26 13:35:52,606 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 13:35:52,607 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-26T13:35:52,650 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-26 13:35:52,655 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 13:35:52,656 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T13:35:53,098 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23 2024-11-26T13:35:53,106 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-26T13:35:53,109 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:35:53,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T13:35:53,136 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd, deleteOnExit=true 2024-11-26T13:35:53,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T13:35:53,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/test.cache.data in system properties and HBase conf 2024-11-26T13:35:53,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T13:35:53,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir in system properties and HBase conf 2024-11-26T13:35:53,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T13:35:53,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T13:35:53,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T13:35:53,300 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T13:35:53,445 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T13:35:53,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T13:35:53,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T13:35:53,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T13:35:53,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T13:35:53,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T13:35:53,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T13:35:53,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T13:35:53,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T13:35:53,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T13:35:53,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/nfs.dump.dir in system properties and HBase conf 2024-11-26T13:35:53,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/java.io.tmpdir in system properties and HBase conf 2024-11-26T13:35:53,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T13:35:53,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T13:35:53,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T13:35:54,539 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T13:35:54,640 INFO [Time-limited test {}] log.Log(170): Logging initialized @3371ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T13:35:54,757 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:35:54,867 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T13:35:54,922 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T13:35:54,923 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T13:35:54,925 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T13:35:54,951 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:35:54,955 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71f46724{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,AVAILABLE} 2024-11-26T13:35:54,957 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17280962{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T13:35:55,294 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6df1a11{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/java.io.tmpdir/jetty-localhost-41833-hadoop-hdfs-3_4_1-tests_jar-_-any-17097409609688037165/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T13:35:55,303 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34c8bc74{HTTP/1.1, (http/1.1)}{localhost:41833} 2024-11-26T13:35:55,303 INFO [Time-limited test {}] server.Server(415): Started @4035ms 2024-11-26T13:35:55,790 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:35:55,798 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T13:35:55,800 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T13:35:55,800 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T13:35:55,800 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T13:35:55,802 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b9b7476{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,AVAILABLE} 2024-11-26T13:35:55,803 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6be2f62b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T13:35:55,956 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6628e1d0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/java.io.tmpdir/jetty-localhost-35751-hadoop-hdfs-3_4_1-tests_jar-_-any-8626358105278364854/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T13:35:55,957 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d5d982f{HTTP/1.1, (http/1.1)}{localhost:35751} 2024-11-26T13:35:55,957 INFO [Time-limited test {}] server.Server(415): Started @4689ms 2024-11-26T13:35:56,040 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T13:35:56,691 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/data/data1/current/BP-650579929-172.17.0.2-1732628154247/current, will proceed with Du for space computation calculation, 2024-11-26T13:35:56,692 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/data/data2/current/BP-650579929-172.17.0.2-1732628154247/current, will proceed with Du for space computation calculation, 2024-11-26T13:35:56,811 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T13:35:56,899 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23 2024-11-26T13:35:56,905 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1a65cfe34aef4b5b with lease ID 0x2ffddef3fe011351: Processing first storage report for DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124 from datanode DatanodeRegistration(127.0.0.1:46229, datanodeUuid=9ccdf160-6c18-4a5c-9556-fc6dc2fa79c8, infoPort=44203, infoSecurePort=0, ipcPort=36001, storageInfo=lv=-57;cid=testClusterID;nsid=1282648622;c=1732628154247) 2024-11-26T13:35:56,907 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1a65cfe34aef4b5b with lease ID 0x2ffddef3fe011351: from storage DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124 node DatanodeRegistration(127.0.0.1:46229, datanodeUuid=9ccdf160-6c18-4a5c-9556-fc6dc2fa79c8, infoPort=44203, infoSecurePort=0, ipcPort=36001, storageInfo=lv=-57;cid=testClusterID;nsid=1282648622;c=1732628154247), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T13:35:56,907 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1a65cfe34aef4b5b with lease ID 0x2ffddef3fe011351: Processing first storage report for DS-15dacfd8-19f3-4667-a0f1-76a11eb00f9a from datanode DatanodeRegistration(127.0.0.1:46229, datanodeUuid=9ccdf160-6c18-4a5c-9556-fc6dc2fa79c8, infoPort=44203, infoSecurePort=0, ipcPort=36001, storageInfo=lv=-57;cid=testClusterID;nsid=1282648622;c=1732628154247) 2024-11-26T13:35:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1a65cfe34aef4b5b with lease ID 0x2ffddef3fe011351: from storage DS-15dacfd8-19f3-4667-a0f1-76a11eb00f9a node DatanodeRegistration(127.0.0.1:46229, datanodeUuid=9ccdf160-6c18-4a5c-9556-fc6dc2fa79c8, infoPort=44203, infoSecurePort=0, ipcPort=36001, storageInfo=lv=-57;cid=testClusterID;nsid=1282648622;c=1732628154247), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T13:35:57,020 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/zookeeper_0, clientPort=62595, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T13:35:57,032 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62595 2024-11-26T13:35:57,050 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T13:35:57,054 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T13:35:57,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741825_1001 (size=7) 2024-11-26T13:35:57,729 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 with version=8 2024-11-26T13:35:57,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/hbase-staging 2024-11-26T13:35:57,742 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-26T13:35:57,743 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-26T13:35:57,743 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-26T13:35:57,743 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-26T13:35:57,838 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T13:35:58,136 INFO [Time-limited test {}] client.ConnectionUtils(128): master/256175a329f8:0 server-side Connection retries=45 2024-11-26T13:35:58,151 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T13:35:58,151 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T13:35:58,159 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T13:35:58,159 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T13:35:58,159 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T13:35:58,307 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T13:35:58,376 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T13:35:58,386 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T13:35:58,390 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T13:35:58,420 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 24718 (auto-detected) 2024-11-26T13:35:58,421 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T13:35:58,443 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40825 2024-11-26T13:35:58,470 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40825 connecting to ZooKeeper ensemble=127.0.0.1:62595 2024-11-26T13:35:58,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:408250x0, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T13:35:58,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40825-0x100340b4b270000 connected 2024-11-26T13:35:58,563 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T13:35:58,565 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T13:35:58,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-26T13:35:58,606 INFO [Time-limited test {}] log.Log(170): Logging initialized @7338ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T13:35:58,673 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-26T13:35:58,673 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-26T13:35:58,674 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-26T13:35:58,675 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-26T13:35:58,676 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-26T13:35:58,677 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-26T13:35:58,678 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-26T13:35:58,678 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-26T13:35:58,681 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-26T13:35:58,753 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 37977 2024-11-26T13:35:58,755 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-26T13:35:58,775 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-26T13:35:58,777 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7ca98a11{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,AVAILABLE} 2024-11-26T13:35:58,778 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-26T13:35:58,778 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5b460d1c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-26T13:35:58,824 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-26T13:35:58,832 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T13:35:58,833 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T13:35:58,834 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T13:35:58,839 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-26T13:35:59,660 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@56e6bf8{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master} 2024-11-26T13:35:59,666 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@69cb2ebd{HTTP/1.1, (http/1.1)}{0.0.0.0:37977} 2024-11-26T13:35:59,667 INFO [Time-limited test {}] server.Server(415): Started @8398ms 2024-11-26T13:35:59,672 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9, hbase.cluster.distributed=false 2024-11-26T13:35:59,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-26T13:35:59,713 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40825 2024-11-26T13:35:59,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40825 2024-11-26T13:35:59,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40825 2024-11-26T13:35:59,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40825 2024-11-26T13:35:59,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40825 2024-11-26T13:35:59,890 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/256175a329f8:0 server-side Connection retries=45 2024-11-26T13:35:59,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T13:35:59,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T13:35:59,893 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T13:35:59,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T13:35:59,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T13:35:59,897 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T13:35:59,899 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T13:35:59,900 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38807 2024-11-26T13:35:59,902 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38807 connecting to ZooKeeper ensemble=127.0.0.1:62595 2024-11-26T13:35:59,904 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T13:35:59,908 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T13:35:59,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388070x0, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T13:35:59,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38807-0x100340b4b270001 connected 2024-11-26T13:35:59,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-26T13:35:59,922 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-26T13:35:59,923 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-26T13:35:59,923 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-26T13:35:59,923 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-26T13:35:59,924 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-26T13:35:59,925 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-26T13:35:59,925 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-26T13:35:59,926 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-26T13:35:59,926 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-26T13:35:59,929 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 39773 2024-11-26T13:35:59,929 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-26T13:35:59,932 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-26T13:35:59,932 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@13c18cd6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,AVAILABLE} 2024-11-26T13:35:59,933 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-26T13:35:59,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@23a380bc{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-26T13:35:59,949 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-26T13:35:59,951 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T13:35:59,951 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T13:35:59,951 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T13:35:59,953 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-26T13:35:59,958 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@6e12a44d{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-26T13:35:59,959 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5804e71e{HTTP/1.1, (http/1.1)}{0.0.0.0:39773} 2024-11-26T13:35:59,959 INFO [Time-limited test {}] server.Server(415): Started @8691ms 2024-11-26T13:35:59,966 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T13:35:59,978 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T13:35:59,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-26T13:35:59,987 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-26T13:35:59,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38807 2024-11-26T13:35:59,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38807 2024-11-26T13:35:59,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38807 2024-11-26T13:35:59,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38807 2024-11-26T13:35:59,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38807 2024-11-26T13:36:00,016 DEBUG [M:0;256175a329f8:40825 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;256175a329f8:40825 2024-11-26T13:36:00,019 INFO [master/256175a329f8:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-26T13:36:00,024 INFO [master/256175a329f8:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@39c3cce8{HTTP/1.1, (http/1.1)}{0.0.0.0:44639} 2024-11-26T13:36:00,024 INFO [master/256175a329f8:0:becomeActiveMaster {}] server.Server(415): Started @8756ms 2024-11-26T13:36:00,025 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/256175a329f8,40825,1732628157903 2024-11-26T13:36:00,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-26T13:36:00,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-26T13:36:00,037 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/256175a329f8,40825,1732628157903 2024-11-26T13:36:00,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-26T13:36:00,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:00,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:00,071 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-26T13:36:00,073 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/256175a329f8,40825,1732628157903 from backup master directory 2024-11-26T13:36:00,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/256175a329f8,40825,1732628157903 2024-11-26T13:36:00,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-26T13:36:00,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-26T13:36:00,077 WARN [master/256175a329f8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T13:36:00,077 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=256175a329f8,40825,1732628157903 2024-11-26T13:36:00,079 INFO [master/256175a329f8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T13:36:00,081 INFO [master/256175a329f8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T13:36:00,160 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/hbase.id] with ID: 722448d4-7b7c-4464-9023-558761cfe9fd 2024-11-26T13:36:00,161 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.tmp/hbase.id 2024-11-26T13:36:00,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741826_1002 (size=42) 2024-11-26T13:36:00,577 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.tmp/hbase.id]:[hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/hbase.id] 2024-11-26T13:36:00,626 INFO [master/256175a329f8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T13:36:00,633 INFO [master/256175a329f8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T13:36:00,658 INFO [master/256175a329f8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-26T13:36:00,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:00,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:00,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741827_1003 (size=196) 2024-11-26T13:36:01,095 INFO [master/256175a329f8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:01,097 INFO [master/256175a329f8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T13:36:01,104 INFO [master/256175a329f8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-26T13:36:01,107 INFO [master/256175a329f8:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-26T13:36:01,119 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T13:36:01,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741828_1004 (size=1189) 2024-11-26T13:36:01,575 INFO [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store 2024-11-26T13:36:01,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741829_1005 (size=34) 2024-11-26T13:36:02,001 INFO [master/256175a329f8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T13:36:02,005 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:02,007 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T13:36:02,007 INFO [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T13:36:02,007 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T13:36:02,010 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T13:36:02,010 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T13:36:02,010 INFO [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T13:36:02,012 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732628162007Disabling compacts and flushes for region at 1732628162007Disabling writes for close at 1732628162010 (+3 ms)Writing region close event to WAL at 1732628162010Closed at 1732628162010 2024-11-26T13:36:02,015 WARN [master/256175a329f8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/.initializing 2024-11-26T13:36:02,015 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/WALs/256175a329f8,40825,1732628157903 2024-11-26T13:36:02,022 INFO [master/256175a329f8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T13:36:02,028 INFO [master/256175a329f8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 256175a329f8,40825,1732628157903.regiongroup-0 2024-11-26T13:36:02,050 INFO [master/256175a329f8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=256175a329f8%2C40825%2C1732628157903.256175a329f8%2C40825%2C1732628157903.regiongroup-0, suffix=, logDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/WALs/256175a329f8,40825,1732628157903, archiveDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/oldWALs, maxLogs=10 2024-11-26T13:36:02,087 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/WALs/256175a329f8,40825,1732628157903/256175a329f8%2C40825%2C1732628157903.256175a329f8%2C40825%2C1732628157903.regiongroup-0.1732628162056, exclude list is [], retry=0 2024-11-26T13:36:02,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:02,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T13:36:02,161 INFO [master/256175a329f8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/WALs/256175a329f8,40825,1732628157903/256175a329f8%2C40825%2C1732628157903.256175a329f8%2C40825%2C1732628157903.regiongroup-0.1732628162056 2024-11-26T13:36:02,162 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:02,163 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:02,164 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:02,167 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,168 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,287 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T13:36:02,293 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:02,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,303 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T13:36:02,303 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:02,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,309 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T13:36:02,310 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,311 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:02,311 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,315 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T13:36:02,315 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,320 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:02,320 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,324 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,326 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,333 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,334 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,338 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T13:36:02,344 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T13:36:02,350 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T13:36:02,351 INFO [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9751069440, jitterRate=-0.09186089038848877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T13:36:02,358 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732628162198Initializing all the Stores at 1732628162201 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732628162202 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628162210 (+8 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628162211 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628162211Cleaning up temporary data from old regions at 1732628162334 (+123 ms)Region opened successfully at 1732628162358 (+24 ms) 2024-11-26T13:36:02,360 INFO [master/256175a329f8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T13:36:02,399 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f748ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=256175a329f8/172.17.0.2:0 2024-11-26T13:36:02,445 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T13:36:02,459 INFO [master/256175a329f8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T13:36:02,459 INFO [master/256175a329f8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T13:36:02,463 INFO [master/256175a329f8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T13:36:02,465 INFO [master/256175a329f8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T13:36:02,470 INFO [master/256175a329f8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T13:36:02,471 INFO [master/256175a329f8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T13:36:02,500 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T13:36:02,512 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-26T13:36:02,515 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-26T13:36:02,518 INFO [master/256175a329f8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T13:36:02,520 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-26T13:36:02,522 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-26T13:36:02,525 INFO [master/256175a329f8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T13:36:02,530 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-26T13:36:02,532 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-26T13:36:02,534 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-26T13:36:02,536 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-26T13:36:02,560 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T13:36:02,562 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-26T13:36:02,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-26T13:36:02,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-26T13:36:02,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:02,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:02,571 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=256175a329f8,40825,1732628157903, sessionid=0x100340b4b270000, setting cluster-up flag (Was=false) 2024-11-26T13:36:02,579 INFO [master/256175a329f8:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-26T13:36:02,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:02,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:02,620 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-26T13:36:02,623 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=256175a329f8,40825,1732628157903 2024-11-26T13:36:02,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:02,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:02,640 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-26T13:36:02,642 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=256175a329f8,40825,1732628157903 2024-11-26T13:36:02,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:02,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:02,661 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-26T13:36:02,662 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=256175a329f8,40825,1732628157903 2024-11-26T13:36:02,669 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T13:36:02,708 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(746): ClusterId : 722448d4-7b7c-4464-9023-558761cfe9fd 2024-11-26T13:36:02,709 INFO [RS:0;256175a329f8:38807 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-26T13:36:02,711 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T13:36:02,717 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T13:36:02,717 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-26T13:36:02,726 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-26T13:36:02,726 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T13:36:02,730 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T13:36:02,731 DEBUG [RS:0;256175a329f8:38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3997a82c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=256175a329f8/172.17.0.2:0 2024-11-26T13:36:02,751 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;256175a329f8:38807 2024-11-26T13:36:02,755 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T13:36:02,756 INFO [RS:0;256175a329f8:38807 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T13:36:02,757 INFO [RS:0;256175a329f8:38807 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T13:36:02,757 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T13:36:02,760 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(2659): reportForDuty to master=256175a329f8,40825,1732628157903 with port=38807, startcode=1732628159830 2024-11-26T13:36:02,765 INFO [master/256175a329f8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T13:36:02,771 INFO [master/256175a329f8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T13:36:02,771 DEBUG [RS:0;256175a329f8:38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T13:36:02,776 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 256175a329f8,40825,1732628157903 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T13:36:02,783 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/256175a329f8:0, corePoolSize=5, maxPoolSize=5 2024-11-26T13:36:02,783 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/256175a329f8:0, corePoolSize=5, maxPoolSize=5 2024-11-26T13:36:02,783 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/256175a329f8:0, corePoolSize=5, maxPoolSize=5 2024-11-26T13:36:02,784 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/256175a329f8:0, corePoolSize=5, maxPoolSize=5 2024-11-26T13:36:02,784 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/256175a329f8:0, corePoolSize=10, maxPoolSize=10 2024-11-26T13:36:02,784 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:02,784 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/256175a329f8:0, corePoolSize=2, maxPoolSize=2 2024-11-26T13:36:02,784 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:02,792 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T13:36:02,792 INFO [master/256175a329f8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732628192792 2024-11-26T13:36:02,793 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T13:36:02,794 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T13:36:02,795 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T13:36:02,800 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,801 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T13:36:02,801 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T13:36:02,802 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-26T13:36:02,803 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T13:36:02,803 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T13:36:02,803 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T13:36:02,804 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:02,812 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T13:36:02,814 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T13:36:02,815 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34359, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T13:36:02,815 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T13:36:02,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741831_1007 (size=1321) 2024-11-26T13:36:02,821 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T13:36:02,822 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:02,823 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40825 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T13:36:02,828 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d5b5b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:02,828 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:02,833 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:02,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741832_1008 (size=32) 2024-11-26T13:36:02,854 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:02,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:02,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:02,861 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T13:36:02,862 WARN [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T13:36:02,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:02,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5e0dfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:02,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:02,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T13:36:02,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:02,867 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T13:36:02,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:02,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T13:36:02,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:02,870 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:02,875 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T13:36:02,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af1e69a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:02,875 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,875 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-26T13:36:02,878 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T13:36:02,878 INFO [master/256175a329f8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T13:36:02,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:02,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T13:36:02,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T13:36:02,883 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,884 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/256175a329f8:0:becomeActiveMaster-HFileCleaner.large.0-1732628162880,5,FailOnTimeoutGroup] 2024-11-26T13:36:02,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:02,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T13:36:02,885 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/256175a329f8:0:becomeActiveMaster-HFileCleaner.small.0-1732628162884,5,FailOnTimeoutGroup] 2024-11-26T13:36:02,885 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:02,885 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T13:36:02,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T13:36:02,887 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:02,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:02,888 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:02,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:02,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T13:36:02,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740 2024-11-26T13:36:02,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740 2024-11-26T13:36:02,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T13:36:02,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T13:36:02,895 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T13:36:02,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T13:36:02,906 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T13:36:02,908 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10424936480, jitterRate=-0.029102131724357605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T13:36:02,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732628162861Initializing all the Stores at 1732628162863 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732628162863Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732628162864 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628162864Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732628162864Cleaning up temporary data from old regions at 1732628162894 (+30 ms)Region opened successfully at 1732628162912 (+18 ms) 2024-11-26T13:36:02,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T13:36:02,912 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T13:36:02,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T13:36:02,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T13:36:02,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T13:36:02,916 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T13:36:02,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732628162912Disabling compacts and flushes for region at 1732628162912Disabling writes for close at 1732628162912Writing region close event to WAL at 1732628162916 (+4 ms)Closed at 1732628162916 2024-11-26T13:36:02,922 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T13:36:02,922 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T13:36:02,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T13:36:02,943 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T13:36:02,947 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T13:36:02,963 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(2659): reportForDuty to master=256175a329f8,40825,1732628157903 with port=38807, startcode=1732628159830 2024-11-26T13:36:02,967 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40825 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 256175a329f8,38807,1732628159830 2024-11-26T13:36:02,970 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40825 {}] master.ServerManager(517): Registering regionserver=256175a329f8,38807,1732628159830 2024-11-26T13:36:02,979 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:02,979 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36063 2024-11-26T13:36:02,979 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=37977 2024-11-26T13:36:02,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-26T13:36:02,988 DEBUG [RS:0;256175a329f8:38807 {}] zookeeper.ZKUtil(111): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rs/256175a329f8,38807,1732628159830 2024-11-26T13:36:02,988 WARN [RS:0;256175a329f8:38807 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T13:36:02,988 INFO [RS:0;256175a329f8:38807 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-26T13:36:02,989 INFO [RS:0;256175a329f8:38807 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-26T13:36:02,989 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830 2024-11-26T13:36:02,990 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [256175a329f8,38807,1732628159830] 2024-11-26T13:36:03,015 INFO [RS:0;256175a329f8:38807 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T13:36:03,028 INFO [RS:0;256175a329f8:38807 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T13:36:03,034 INFO [RS:0;256175a329f8:38807 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T13:36:03,034 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,035 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T13:36:03,041 INFO [RS:0;256175a329f8:38807 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T13:36:03,043 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,043 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,043 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,043 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,043 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,044 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,044 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/256175a329f8:0, corePoolSize=2, maxPoolSize=2 2024-11-26T13:36:03,044 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,044 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,044 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,045 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,045 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,045 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/256175a329f8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T13:36:03,045 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0, corePoolSize=3, maxPoolSize=3 2024-11-26T13:36:03,045 DEBUG [RS:0;256175a329f8:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/256175a329f8:0, corePoolSize=3, maxPoolSize=3 2024-11-26T13:36:03,046 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,046 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,046 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,046 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,047 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,047 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=256175a329f8,38807,1732628159830-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T13:36:03,064 INFO [RS:0;256175a329f8:38807 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T13:36:03,066 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=256175a329f8,38807,1732628159830-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,067 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,067 INFO [RS:0;256175a329f8:38807 {}] regionserver.Replication(171): 256175a329f8,38807,1732628159830 started 2024-11-26T13:36:03,084 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,084 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1482): Serving as 256175a329f8,38807,1732628159830, RpcServer on 256175a329f8/172.17.0.2:38807, sessionid=0x100340b4b270001 2024-11-26T13:36:03,085 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T13:36:03,085 DEBUG [RS:0;256175a329f8:38807 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 256175a329f8,38807,1732628159830 2024-11-26T13:36:03,085 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '256175a329f8,38807,1732628159830' 2024-11-26T13:36:03,085 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-26T13:36:03,086 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-26T13:36:03,087 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T13:36:03,087 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-26T13:36:03,087 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '256175a329f8,38807,1732628159830' 2024-11-26T13:36:03,087 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:36:03,088 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:36:03,088 INFO [RS:0;256175a329f8:38807 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-26T13:36:03,088 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-26T13:36:03,088 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T13:36:03,088 DEBUG [RS:0;256175a329f8:38807 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 256175a329f8,38807,1732628159830 2024-11-26T13:36:03,089 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '256175a329f8,38807,1732628159830' 2024-11-26T13:36:03,089 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-26T13:36:03,089 DEBUG [RS:0;256175a329f8:38807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-26T13:36:03,090 DEBUG [RS:0;256175a329f8:38807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T13:36:03,090 INFO [RS:0;256175a329f8:38807 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T13:36:03,090 INFO [RS:0;256175a329f8:38807 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T13:36:03,097 WARN [256175a329f8:40825 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T13:36:03,195 INFO [RS:0;256175a329f8:38807 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T13:36:03,196 INFO [RS:0;256175a329f8:38807 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 256175a329f8,38807,1732628159830.regiongroup-0 2024-11-26T13:36:03,199 INFO [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0, suffix=, logDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830, archiveDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs, maxLogs=32 2024-11-26T13:36:03,215 DEBUG [RS:0;256175a329f8:38807 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628163201, exclude list is [], retry=0 2024-11-26T13:36:03,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:03,222 INFO [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628163201 2024-11-26T13:36:03,223 DEBUG [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:03,351 DEBUG [256175a329f8:40825 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T13:36:03,367 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:03,374 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 256175a329f8,38807,1732628159830, state=OPENING 2024-11-26T13:36:03,380 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T13:36:03,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:03,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:36:03,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-26T13:36:03,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-26T13:36:03,385 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T13:36:03,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:03,573 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T13:36:03,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34663, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T13:36:03,589 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T13:36:03,590 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-26T13:36:03,590 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-26T13:36:03,590 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T13:36:03,591 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T13:36:03,595 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=256175a329f8%2C38807%2C1732628159830.meta, suffix=.meta, logDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830, archiveDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs, maxLogs=32 2024-11-26T13:36:03,618 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628163597.meta, exclude list is [], retry=0 2024-11-26T13:36:03,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:03,628 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628163597.meta 2024-11-26T13:36:03,628 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:03,629 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:03,634 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:03,635 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T13:36:03,639 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T13:36:03,639 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T13:36:03,644 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T13:36:03,645 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:03,645 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T13:36:03,645 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T13:36:03,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T13:36:03,651 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T13:36:03,651 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:03,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:03,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T13:36:03,655 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T13:36:03,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:03,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:03,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T13:36:03,658 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T13:36:03,658 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:03,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:03,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T13:36:03,660 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T13:36:03,661 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:03,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T13:36:03,662 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T13:36:03,663 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740 2024-11-26T13:36:03,666 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740 2024-11-26T13:36:03,668 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T13:36:03,668 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T13:36:03,669 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T13:36:03,672 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T13:36:03,674 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10507486560, jitterRate=-0.021414056420326233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T13:36:03,674 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T13:36:03,676 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732628163646Writing region info on filesystem at 1732628163646Initializing all the Stores at 1732628163649 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732628163649Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732628163649Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628163649Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732628163649Cleaning up temporary data from old regions at 1732628163668 (+19 ms)Running coprocessor post-open hooks at 1732628163674 (+6 ms)Region opened successfully at 1732628163676 (+2 ms) 2024-11-26T13:36:03,686 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732628163560 2024-11-26T13:36:03,701 DEBUG [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T13:36:03,701 INFO [RS_OPEN_META-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T13:36:03,703 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:03,706 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 256175a329f8,38807,1732628159830, state=OPEN 2024-11-26T13:36:03,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-26T13:36:03,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-26T13:36:03,709 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-26T13:36:03,709 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-26T13:36:03,710 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=256175a329f8,38807,1732628159830 2024-11-26T13:36:03,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T13:36:03,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=256175a329f8,38807,1732628159830 in 323 msec 2024-11-26T13:36:03,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T13:36:03,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 786 msec 2024-11-26T13:36:03,725 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T13:36:03,725 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T13:36:03,754 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:03,757 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:03,789 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:03,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49029, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:03,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1140 sec 2024-11-26T13:36:03,818 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732628163818, completionTime=-1 2024-11-26T13:36:03,822 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T13:36:03,822 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T13:36:03,858 INFO [master/256175a329f8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T13:36:03,858 INFO [master/256175a329f8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732628223858 2024-11-26T13:36:03,858 INFO [master/256175a329f8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732628283858 2024-11-26T13:36:03,858 INFO [master/256175a329f8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-11-26T13:36:03,861 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=256175a329f8,40825,1732628157903-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,861 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=256175a329f8,40825,1732628157903-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,862 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=256175a329f8,40825,1732628157903-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,864 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-256175a329f8:40825, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,864 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,865 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:03,874 DEBUG [master/256175a329f8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T13:36:03,906 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.829sec 2024-11-26T13:36:03,908 INFO [master/256175a329f8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T13:36:03,909 INFO [master/256175a329f8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T13:36:03,911 INFO [master/256175a329f8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T13:36:03,911 INFO [master/256175a329f8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T13:36:03,911 INFO [master/256175a329f8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T13:36:03,912 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=256175a329f8,40825,1732628157903-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T13:36:03,913 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=256175a329f8,40825,1732628157903-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T13:36:03,923 DEBUG [master/256175a329f8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T13:36:03,924 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T13:36:03,924 INFO [master/256175a329f8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=256175a329f8,40825,1732628157903-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T13:36:04,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b200574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:04,014 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T13:36:04,014 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T13:36:04,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:04,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:04,028 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29567660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:04,034 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:04,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ea5228, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:04,036 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:04,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:04,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:04,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=256175a329f8,40825,1732628157903 2024-11-26T13:36:04,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-26T13:36:04,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/test.cache.data in system properties and HBase conf 2024-11-26T13:36:04,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T13:36:04,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir in system properties and HBase conf 2024-11-26T13:36:04,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T13:36:04,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T13:36:04,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T13:36:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T13:36:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T13:36:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T13:36:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T13:36:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T13:36:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T13:36:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T13:36:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T13:36:04,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T13:36:04,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/nfs.dump.dir in system properties and HBase conf 2024-11-26T13:36:04,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/java.io.tmpdir in system properties and HBase conf 2024-11-26T13:36:04,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T13:36:04,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T13:36:04,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T13:36:04,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741835_1011 (size=592039) 2024-11-26T13:36:04,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741836_1012 (size=1663647) 2024-11-26T13:36:05,770 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:36:05,862 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:36:06,095 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T13:36:06,097 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T13:36:06,097 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T13:36:06,097 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T13:36:06,098 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@485c5160{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,AVAILABLE} 2024-11-26T13:36:06,098 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5bc8e33d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-26T13:36:06,100 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-26T13:36:06,101 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T13:36:06,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T13:36:06,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T13:36:06,107 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T13:36:06,110 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:36:06,122 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@339da2c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,AVAILABLE} 2024-11-26T13:36:06,122 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11a358cb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-26T13:36:06,252 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2024-11-26T13:36:06,252 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2024-11-26T13:36:06,252 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-26T13:36:06,254 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-26T13:36:06,295 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:06,395 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:06,607 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:06,626 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@414bbc4c{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/java.io.tmpdir/jetty-localhost-35441-hadoop-yarn-common-3_4_1_jar-_-any-10799153489242452597/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-26T13:36:06,626 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3a4583c9{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/java.io.tmpdir/jetty-localhost-36783-hadoop-yarn-common-3_4_1_jar-_-any-4628300529986298311/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-26T13:36:06,627 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@6d50a15e{HTTP/1.1, (http/1.1)}{localhost:36783} 2024-11-26T13:36:06,627 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@382d96c0{HTTP/1.1, (http/1.1)}{localhost:35441} 2024-11-26T13:36:06,627 INFO [Thread-251 {}] server.Server(415): Started @15359ms 2024-11-26T13:36:06,627 INFO [Time-limited test {}] server.Server(415): Started @15359ms 2024-11-26T13:36:06,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741837_1013 (size=5) 2024-11-26T13:36:07,869 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-26T13:36:07,874 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:36:07,904 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-26T13:36:07,905 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T13:36:07,907 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T13:36:07,907 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T13:36:07,907 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T13:36:07,908 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:36:07,908 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3476a55f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,AVAILABLE} 2024-11-26T13:36:07,909 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c406c80{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-26T13:36:07,970 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-26T13:36:07,970 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-26T13:36:07,970 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-26T13:36:07,970 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-26T13:36:07,979 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:07,985 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:08,087 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:08,098 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@128c7511{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/java.io.tmpdir/jetty-localhost-37755-hadoop-yarn-common-3_4_1_jar-_-any-3376644346593497041/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-26T13:36:08,100 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d8527a6{HTTP/1.1, (http/1.1)}{localhost:37755} 2024-11-26T13:36:08,100 INFO [Time-limited test {}] server.Server(415): Started @16831ms 2024-11-26T13:36:08,245 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-26T13:36:08,247 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:36:08,260 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-26T13:36:08,261 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T13:36:08,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T13:36:08,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T13:36:08,263 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T13:36:08,263 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T13:36:08,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54c34ba5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,AVAILABLE} 2024-11-26T13:36:08,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42fc61ef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-26T13:36:08,317 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-26T13:36:08,317 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-26T13:36:08,317 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-26T13:36:08,317 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-26T13:36:08,325 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:08,328 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:08,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T13:36:08,374 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T13:36:08,384 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:36:08,384 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-26T13:36:08,385 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T13:36:08,385 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T13:36:08,385 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T13:36:08,385 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T13:36:08,415 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-26T13:36:08,421 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49fbadf3{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/java.io.tmpdir/jetty-localhost-33235-hadoop-yarn-common-3_4_1_jar-_-any-10655736257565225386/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-26T13:36:08,422 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@67020615{HTTP/1.1, (http/1.1)}{localhost:33235} 2024-11-26T13:36:08,422 INFO [Time-limited test {}] server.Server(415): Started @17154ms 2024-11-26T13:36:08,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-26T13:36:08,449 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:36063/backupUT 2024-11-26T13:36:08,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T13:36:08,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 256175a329f8,40825,1732628157903 2024-11-26T13:36:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62289390 2024-11-26T13:36:08,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T13:36:08,465 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T13:36:08,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-11-26T13:36:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-26T13:36:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T13:36:08,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 20 msec 2024-11-26T13:36:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T13:36:08,595 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-26T13:36:08,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-11-26T13:36:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-26T13:36:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-26T13:36:08,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 12 msec 2024-11-26T13:36:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-26T13:36:08,714 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-26T13:36:08,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-11-26T13:36:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-26T13:36:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-26T13:36:08,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 14 msec 2024-11-26T13:36:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-26T13:36:08,824 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-26T13:36:08,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-11-26T13:36:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-26T13:36:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T13:36:08,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 13 msec 2024-11-26T13:36:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T13:36:08,935 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-26T13:36:08,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1732628168449', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1732628168449 2024-11-26T13:36:08,960 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1732628168449 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T13:36:08,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1732628168449" procId is: 8 2024-11-26T13:36:08,963 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-26T13:36:08,965 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1732628168449 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T13:36:08,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741838_1014 (size=367) 2024-11-26T13:36:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-26T13:36:09,167 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-26T13:36:09,219 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T13:36:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-26T13:36:09,382 INFO [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d27e55c185bde71afb21ebc132e1a4f, NAME => 'test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1732628168449', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:09,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741839_1015 (size=53) 2024-11-26T13:36:09,393 DEBUG [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(898): Instantiated test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:09,393 DEBUG [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(1722): Closing 1d27e55c185bde71afb21ebc132e1a4f, disabling compactions & flushes 2024-11-26T13:36:09,393 INFO [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(1755): Closing region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:09,393 DEBUG [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:09,393 DEBUG [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. after waiting 0 ms 2024-11-26T13:36:09,394 DEBUG [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:09,394 INFO [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(1973): Closed test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:09,394 DEBUG [RegionOpenAndInit-test-1732628168449-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d27e55c185bde71afb21ebc132e1a4f: Waiting for close lock at 1732628169393Disabling compacts and flushes for region at 1732628169393Disabling writes for close at 1732628169393Writing region close event to WAL at 1732628169394 (+1 ms)Closed at 1732628169394 2024-11-26T13:36:09,396 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1732628168449 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T13:36:09,402 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732628169396"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732628169396"}]},"ts":"1732628169396"} 2024-11-26T13:36:09,407 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T13:36:09,409 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1732628168449 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T13:36:09,412 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732628168449","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628169409"}]},"ts":"1732628169409"} 2024-11-26T13:36:09,416 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732628168449, state=ENABLING in hbase:meta 2024-11-26T13:36:09,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, ASSIGN}] 2024-11-26T13:36:09,420 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, ASSIGN 2024-11-26T13:36:09,421 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, ASSIGN; state=OFFLINE, location=256175a329f8,38807,1732628159830; forceNewPlan=false, retain=false 2024-11-26T13:36:09,573 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:09,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, ASSIGN because future has completed 2024-11-26T13:36:09,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-26T13:36:09,755 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:09,756 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T13:36:09,756 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor 256175a329f8,38807,1732628159830.regiongroup-1 2024-11-26T13:36:09,759 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1, suffix=, logDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830, archiveDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs, maxLogs=32 2024-11-26T13:36:09,779 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628169762, exclude list is [], retry=0 2024-11-26T13:36:09,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:09,786 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628169762 2024-11-26T13:36:09,786 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:09,786 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d27e55c185bde71afb21ebc132e1a4f, NAME => 'test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:09,787 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:09,787 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732628168449 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,787 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:09,787 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,787 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,790 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,792 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d27e55c185bde71afb21ebc132e1a4f columnFamilyName f 2024-11-26T13:36:09,792 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:09,793 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(327): Store=1d27e55c185bde71afb21ebc132e1a4f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:09,793 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,794 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,795 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,796 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,796 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,799 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,802 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T13:36:09,803 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 1d27e55c185bde71afb21ebc132e1a4f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10091768480, jitterRate=-0.06013081967830658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T13:36:09,803 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:09,805 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 1d27e55c185bde71afb21ebc132e1a4f: Running coprocessor pre-open hook at 1732628169787Writing region info on filesystem at 1732628169788 (+1 ms)Initializing all the Stores at 1732628169789 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628169789Cleaning up temporary data from old regions at 1732628169796 (+7 ms)Running coprocessor post-open hooks at 1732628169803 (+7 ms)Region opened successfully at 1732628169805 (+2 ms) 2024-11-26T13:36:09,806 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., pid=10, masterSystemTime=1732628169733 2024-11-26T13:36:09,810 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:09,810 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:09,811 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=OPEN, openSeqNum=2, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:09,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:09,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T13:36:09,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 in 238 msec 2024-11-26T13:36:09,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-26T13:36:09,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, ASSIGN in 403 msec 2024-11-26T13:36:09,827 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1732628168449 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T13:36:09,828 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732628168449","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628169827"}]},"ts":"1732628169827"} 2024-11-26T13:36:09,831 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732628168449, state=ENABLED in hbase:meta 2024-11-26T13:36:09,832 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1732628168449 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T13:36:09,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1732628168449 in 881 msec 2024-11-26T13:36:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-26T13:36:10,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1732628168449 completed 2024-11-26T13:36:10,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c9f2c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:10,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:10,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:10,107 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3838a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:10,110 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f1fc35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:10,114 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:10,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732628168449', row='row0', locateType=CURRENT is [region=test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., hostname=256175a329f8,38807,1732628159830, seqNum=2] 2024-11-26T13:36:10,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(8528): writing data to region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. with WAL disabled. Data may be lost in the event of a crash. 2024-11-26T13:36:10,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17326281684491', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17326281684491 2024-11-26T13:36:10,283 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17326281684491 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T13:36:10,283 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:10,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17326281684491" procId is: 11 2024-11-26T13:36:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T13:36:10,285 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17326281684491 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T13:36:10,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741841_1017 (size=364) 2024-11-26T13:36:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T13:36:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T13:36:10,698 INFO [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d333b64a51b935fe3679e64e25b2142a, NAME => 'ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17326281684491', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:10,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741842_1018 (size=50) 2024-11-26T13:36:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T13:36:11,108 DEBUG [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:11,109 DEBUG [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(1722): Closing d333b64a51b935fe3679e64e25b2142a, disabling compactions & flushes 2024-11-26T13:36:11,109 INFO [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:11,109 DEBUG [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:11,109 DEBUG [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. after waiting 0 ms 2024-11-26T13:36:11,109 DEBUG [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:11,109 INFO [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:11,109 DEBUG [RegionOpenAndInit-ns2:test-17326281684491-pool-0 {}] regionserver.HRegion(1676): Region close journal for d333b64a51b935fe3679e64e25b2142a: Waiting for close lock at 1732628171108Disabling compacts and flushes for region at 1732628171108Disabling writes for close at 1732628171109 (+1 ms)Writing region close event to WAL at 1732628171109Closed at 1732628171109 2024-11-26T13:36:11,111 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17326281684491 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T13:36:11,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732628171111"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732628171111"}]},"ts":"1732628171111"} 2024-11-26T13:36:11,119 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T13:36:11,121 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17326281684491 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T13:36:11,121 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17326281684491","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628171121"}]},"ts":"1732628171121"} 2024-11-26T13:36:11,123 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17326281684491, state=ENABLING in hbase:meta 2024-11-26T13:36:11,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17326281684491, region=d333b64a51b935fe3679e64e25b2142a, ASSIGN}] 2024-11-26T13:36:11,125 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17326281684491, region=d333b64a51b935fe3679e64e25b2142a, ASSIGN 2024-11-26T13:36:11,127 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17326281684491, region=d333b64a51b935fe3679e64e25b2142a, ASSIGN; state=OFFLINE, location=256175a329f8,38807,1732628159830; forceNewPlan=false, retain=false 2024-11-26T13:36:11,277 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=d333b64a51b935fe3679e64e25b2142a, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:11,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17326281684491, region=d333b64a51b935fe3679e64e25b2142a, ASSIGN because future has completed 2024-11-26T13:36:11,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure d333b64a51b935fe3679e64e25b2142a, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T13:36:11,441 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:11,441 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => d333b64a51b935fe3679e64e25b2142a, NAME => 'ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:11,442 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:11,442 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17326281684491 d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,442 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:11,442 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,442 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,444 INFO [StoreOpener-d333b64a51b935fe3679e64e25b2142a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,447 INFO [StoreOpener-d333b64a51b935fe3679e64e25b2142a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d333b64a51b935fe3679e64e25b2142a columnFamilyName f 2024-11-26T13:36:11,447 DEBUG [StoreOpener-d333b64a51b935fe3679e64e25b2142a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:11,448 INFO [StoreOpener-d333b64a51b935fe3679e64e25b2142a-1 {}] regionserver.HStore(327): Store=d333b64a51b935fe3679e64e25b2142a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:11,448 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,450 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,450 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,451 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,451 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,455 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,458 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T13:36:11,459 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened d333b64a51b935fe3679e64e25b2142a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10084198560, jitterRate=-0.0608358234167099}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T13:36:11,459 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:11,461 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for d333b64a51b935fe3679e64e25b2142a: Running coprocessor pre-open hook at 1732628171442Writing region info on filesystem at 1732628171442Initializing all the Stores at 1732628171444 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628171444Cleaning up temporary data from old regions at 1732628171451 (+7 ms)Running coprocessor post-open hooks at 1732628171459 (+8 ms)Region opened successfully at 1732628171461 (+2 ms) 2024-11-26T13:36:11,463 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a., pid=13, masterSystemTime=1732628171435 2024-11-26T13:36:11,466 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:11,466 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:11,467 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=d333b64a51b935fe3679e64e25b2142a, regionState=OPEN, openSeqNum=2, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:11,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure d333b64a51b935fe3679e64e25b2142a, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:11,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-26T13:36:11,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure d333b64a51b935fe3679e64e25b2142a, server=256175a329f8,38807,1732628159830 in 190 msec 2024-11-26T13:36:11,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T13:36:11,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17326281684491, region=d333b64a51b935fe3679e64e25b2142a, ASSIGN in 352 msec 2024-11-26T13:36:11,481 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17326281684491 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T13:36:11,481 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17326281684491","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628171481"}]},"ts":"1732628171481"} 2024-11-26T13:36:11,484 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17326281684491, state=ENABLED in hbase:meta 2024-11-26T13:36:11,485 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17326281684491 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T13:36:11,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17326281684491 in 1.2050 sec 2024-11-26T13:36:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T13:36:12,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17326281684491 completed 2024-11-26T13:36:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17326281684491', row='row0', locateType=CURRENT is [region=ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a., hostname=256175a329f8,38807,1732628159830, seqNum=2] 2024-11-26T13:36:12,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(8528): writing data to region ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. with WAL disabled. Data may be lost in the event of a crash. 2024-11-26T13:36:12,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17326281684492', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17326281684492 2024-11-26T13:36:12,614 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17326281684492 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T13:36:12,614 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:12,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17326281684492" procId is: 14 2024-11-26T13:36:12,616 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17326281684492 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T13:36:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-26T13:36:12,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741843_1019 (size=390) 2024-11-26T13:36:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-26T13:36:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-26T13:36:13,028 INFO [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a62bf88ee852e800513fdadb466dea7e, NAME => 'ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17326281684492', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:13,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741844_1020 (size=50) 2024-11-26T13:36:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-26T13:36:13,438 DEBUG [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:13,438 DEBUG [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(1722): Closing a62bf88ee852e800513fdadb466dea7e, disabling compactions & flushes 2024-11-26T13:36:13,438 INFO [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:36:13,438 DEBUG [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:36:13,438 DEBUG [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. after waiting 0 ms 2024-11-26T13:36:13,438 DEBUG [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:36:13,438 INFO [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:36:13,438 DEBUG [RegionOpenAndInit-ns3:test-17326281684492-pool-0 {}] regionserver.HRegion(1676): Region close journal for a62bf88ee852e800513fdadb466dea7e: Waiting for close lock at 1732628173438Disabling compacts and flushes for region at 1732628173438Disabling writes for close at 1732628173438Writing region close event to WAL at 1732628173438Closed at 1732628173438 2024-11-26T13:36:13,440 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17326281684492 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T13:36:13,441 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732628173440"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732628173440"}]},"ts":"1732628173440"} 2024-11-26T13:36:13,444 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T13:36:13,445 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17326281684492 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T13:36:13,445 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17326281684492","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628173445"}]},"ts":"1732628173445"} 2024-11-26T13:36:13,448 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17326281684492, state=ENABLING in hbase:meta 2024-11-26T13:36:13,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17326281684492, region=a62bf88ee852e800513fdadb466dea7e, ASSIGN}] 2024-11-26T13:36:13,451 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17326281684492, region=a62bf88ee852e800513fdadb466dea7e, ASSIGN 2024-11-26T13:36:13,452 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17326281684492, region=a62bf88ee852e800513fdadb466dea7e, ASSIGN; state=OFFLINE, location=256175a329f8,38807,1732628159830; forceNewPlan=false, retain=false 2024-11-26T13:36:13,603 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a62bf88ee852e800513fdadb466dea7e, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:13,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17326281684492, region=a62bf88ee852e800513fdadb466dea7e, ASSIGN because future has completed 2024-11-26T13:36:13,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62bf88ee852e800513fdadb466dea7e, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-26T13:36:13,766 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:36:13,767 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => a62bf88ee852e800513fdadb466dea7e, NAME => 'ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:13,767 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:13,767 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17326281684492 a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,767 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:13,767 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,768 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,770 INFO [StoreOpener-a62bf88ee852e800513fdadb466dea7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,772 INFO [StoreOpener-a62bf88ee852e800513fdadb466dea7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a62bf88ee852e800513fdadb466dea7e columnFamilyName f 2024-11-26T13:36:13,772 DEBUG [StoreOpener-a62bf88ee852e800513fdadb466dea7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:13,773 INFO [StoreOpener-a62bf88ee852e800513fdadb466dea7e-1 {}] regionserver.HStore(327): Store=a62bf88ee852e800513fdadb466dea7e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:13,773 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,774 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns3/test-17326281684492/a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,775 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns3/test-17326281684492/a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,775 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,775 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,778 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,781 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns3/test-17326281684492/a62bf88ee852e800513fdadb466dea7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T13:36:13,782 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened a62bf88ee852e800513fdadb466dea7e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10616824960, jitterRate=-0.01123112440109253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T13:36:13,782 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:36:13,783 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for a62bf88ee852e800513fdadb466dea7e: Running coprocessor pre-open hook at 1732628173768Writing region info on filesystem at 1732628173768Initializing all the Stores at 1732628173769 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628173769Cleaning up temporary data from old regions at 1732628173775 (+6 ms)Running coprocessor post-open hooks at 1732628173782 (+7 ms)Region opened successfully at 1732628173783 (+1 ms) 2024-11-26T13:36:13,785 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e., pid=16, masterSystemTime=1732628173760 2024-11-26T13:36:13,788 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:36:13,788 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:36:13,789 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a62bf88ee852e800513fdadb466dea7e, regionState=OPEN, openSeqNum=2, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:13,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a62bf88ee852e800513fdadb466dea7e, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:13,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-26T13:36:13,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure a62bf88ee852e800513fdadb466dea7e, server=256175a329f8,38807,1732628159830 in 187 msec 2024-11-26T13:36:13,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-26T13:36:13,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17326281684492, region=a62bf88ee852e800513fdadb466dea7e, ASSIGN in 349 msec 2024-11-26T13:36:13,802 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17326281684492 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T13:36:13,802 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17326281684492","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628173802"}]},"ts":"1732628173802"} 2024-11-26T13:36:13,805 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17326281684492, state=ENABLED in hbase:meta 2024-11-26T13:36:13,806 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17326281684492 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T13:36:13,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17326281684492 in 1.1950 sec 2024-11-26T13:36:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-26T13:36:14,774 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17326281684492 completed 2024-11-26T13:36:14,774 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17326281684492 get assigned. Timeout = 60000ms 2024-11-26T13:36:14,776 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T13:36:14,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17326281684492 assigned to meta. Checking AM states. 2024-11-26T13:36:14,782 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T13:36:14,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17326281684492 assigned. 2024-11-26T13:36:14,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17326281684493', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17326281684493 2024-11-26T13:36:14,788 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17326281684493 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T13:36:14,788 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:14,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17326281684493" procId is: 17 2024-11-26T13:36:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T13:36:14,789 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17326281684493 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T13:36:14,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741845_1021 (size=390) 2024-11-26T13:36:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T13:36:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T13:36:15,201 INFO [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 14cd7ab999b32e4f744910187c1c85c5, NAME => 'ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17326281684493', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:15,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741846_1022 (size=50) 2024-11-26T13:36:15,290 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-26T13:36:15,327 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17326281684491' 2024-11-26T13:36:15,328 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17326281684492' 2024-11-26T13:36:15,328 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1732628168449' 2024-11-26T13:36:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T13:36:15,610 DEBUG [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:15,610 DEBUG [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(1722): Closing 14cd7ab999b32e4f744910187c1c85c5, disabling compactions & flushes 2024-11-26T13:36:15,611 INFO [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:36:15,611 DEBUG [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:36:15,611 DEBUG [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. after waiting 0 ms 2024-11-26T13:36:15,611 DEBUG [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:36:15,611 INFO [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:36:15,611 DEBUG [RegionOpenAndInit-ns4:test-17326281684493-pool-0 {}] regionserver.HRegion(1676): Region close journal for 14cd7ab999b32e4f744910187c1c85c5: Waiting for close lock at 1732628175610Disabling compacts and flushes for region at 1732628175610Disabling writes for close at 1732628175611 (+1 ms)Writing region close event to WAL at 1732628175611Closed at 1732628175611 2024-11-26T13:36:15,613 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17326281684493 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T13:36:15,613 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732628175613"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732628175613"}]},"ts":"1732628175613"} 2024-11-26T13:36:15,616 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T13:36:15,617 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17326281684493 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T13:36:15,617 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17326281684493","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628175617"}]},"ts":"1732628175617"} 2024-11-26T13:36:15,620 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17326281684493, state=ENABLING in hbase:meta 2024-11-26T13:36:15,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17326281684493, region=14cd7ab999b32e4f744910187c1c85c5, ASSIGN}] 2024-11-26T13:36:15,621 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17326281684493, region=14cd7ab999b32e4f744910187c1c85c5, ASSIGN 2024-11-26T13:36:15,623 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17326281684493, region=14cd7ab999b32e4f744910187c1c85c5, ASSIGN; state=OFFLINE, location=256175a329f8,38807,1732628159830; forceNewPlan=false, retain=false 2024-11-26T13:36:15,773 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=14cd7ab999b32e4f744910187c1c85c5, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:15,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17326281684493, region=14cd7ab999b32e4f744910187c1c85c5, ASSIGN because future has completed 2024-11-26T13:36:15,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14cd7ab999b32e4f744910187c1c85c5, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T13:36:15,936 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:36:15,937 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => 14cd7ab999b32e4f744910187c1c85c5, NAME => 'ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:15,937 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:15,937 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17326281684493 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,938 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:15,938 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,938 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,940 INFO [StoreOpener-14cd7ab999b32e4f744910187c1c85c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,943 INFO [StoreOpener-14cd7ab999b32e4f744910187c1c85c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14cd7ab999b32e4f744910187c1c85c5 columnFamilyName f 2024-11-26T13:36:15,943 DEBUG [StoreOpener-14cd7ab999b32e4f744910187c1c85c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:15,944 INFO [StoreOpener-14cd7ab999b32e4f744910187c1c85c5-1 {}] regionserver.HStore(327): Store=14cd7ab999b32e4f744910187c1c85c5/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:15,944 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,945 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns4/test-17326281684493/14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,945 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns4/test-17326281684493/14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,946 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,946 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,949 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,952 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns4/test-17326281684493/14cd7ab999b32e4f744910187c1c85c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T13:36:15,953 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened 14cd7ab999b32e4f744910187c1c85c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10367385600, jitterRate=-0.03446197509765625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T13:36:15,953 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:36:15,954 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for 14cd7ab999b32e4f744910187c1c85c5: Running coprocessor pre-open hook at 1732628175938Writing region info on filesystem at 1732628175938Initializing all the Stores at 1732628175939 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628175939Cleaning up temporary data from old regions at 1732628175946 (+7 ms)Running coprocessor post-open hooks at 1732628175953 (+7 ms)Region opened successfully at 1732628175954 (+1 ms) 2024-11-26T13:36:15,955 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5., pid=19, masterSystemTime=1732628175930 2024-11-26T13:36:15,958 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:36:15,958 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:36:15,959 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=14cd7ab999b32e4f744910187c1c85c5, regionState=OPEN, openSeqNum=2, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:15,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14cd7ab999b32e4f744910187c1c85c5, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:15,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-26T13:36:15,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 14cd7ab999b32e4f744910187c1c85c5, server=256175a329f8,38807,1732628159830 in 186 msec 2024-11-26T13:36:15,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-26T13:36:15,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17326281684493, region=14cd7ab999b32e4f744910187c1c85c5, ASSIGN in 346 msec 2024-11-26T13:36:15,971 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17326281684493 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T13:36:15,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17326281684493","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628175971"}]},"ts":"1732628175971"} 2024-11-26T13:36:15,973 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17326281684493, state=ENABLED in hbase:meta 2024-11-26T13:36:15,974 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17326281684493 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T13:36:15,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17326281684493 in 1.1900 sec 2024-11-26T13:36:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T13:36:16,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17326281684493 completed 2024-11-26T13:36:16,934 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17326281684493 get assigned. Timeout = 60000ms 2024-11-26T13:36:16,934 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T13:36:16,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17326281684493 assigned to meta. Checking AM states. 2024-11-26T13:36:16,938 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T13:36:16,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17326281684493 assigned. 2024-11-26T13:36:16,939 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T13:36:16,939 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:36:16,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:16,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:16,945 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:16,977 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=16, AvailableMemoryMB=5331 2024-11-26T13:36:16,979 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-11-26T13:36:16,980 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-26T13:36:16,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732628168449 from 'test-1732628168449', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732628168449', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732628168449 2024-11-26T13:36:17,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741847_1023 (size=908) 2024-11-26T13:36:17,414 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/.tabledesc/.tableinfo.0000000001.367 2024-11-26T13:36:17,414 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/.tabledesc/.tableinfo.0000000002.908 2024-11-26T13:36:17,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732628168449}] 2024-11-26T13:36:17,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE}] 2024-11-26T13:36:17,430 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE 2024-11-26T13:36:17,432 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=CLOSING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:17,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE because future has completed 2024-11-26T13:36:17,435 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T13:36:17,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:17,594 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:17,594 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T13:36:17,595 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 1d27e55c185bde71afb21ebc132e1a4f, disabling compactions & flushes 2024-11-26T13:36:17,595 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:17,595 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:17,595 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. after waiting 0 ms 2024-11-26T13:36:17,595 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:17,599 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 1d27e55c185bde71afb21ebc132e1a4f 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-26T13:36:17,670 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f/a9abd5f0b946478b83e74ccb81c3d433 is 37, key is row10/f:q1/1732628170143/Put/seqid=0 2024-11-26T13:36:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741848_1024 (size=8413) 2024-11-26T13:36:18,082 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f/a9abd5f0b946478b83e74ccb81c3d433 2024-11-26T13:36:18,143 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f/a9abd5f0b946478b83e74ccb81c3d433 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 2024-11-26T13:36:18,153 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433, entries=99, sequenceid=103, filesize=8.2 K 2024-11-26T13:36:18,163 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 1d27e55c185bde71afb21ebc132e1a4f in 561ms, sequenceid=103, compaction requested=false 2024-11-26T13:36:18,171 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-26T13:36:18,174 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:36:18,174 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:18,175 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 1d27e55c185bde71afb21ebc132e1a4f: Waiting for close lock at 1732628177595Running coprocessor pre-close hooks at 1732628177595Disabling compacts and flushes for region at 1732628177595Disabling writes for close at 1732628177595Obtaining lock to block concurrent updates at 1732628177599 (+4 ms)Preparing flush snapshotting stores in 1d27e55c185bde71afb21ebc132e1a4f at 1732628177599Finished memstore snapshotting test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1732628177607 (+8 ms)Flushing stores of test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. at 1732628177608 (+1 ms)Flushing 1d27e55c185bde71afb21ebc132e1a4f/f: creating writer at 1732628177610 (+2 ms)Flushing 1d27e55c185bde71afb21ebc132e1a4f/f: appending metadata at 1732628177658 (+48 ms)Flushing 1d27e55c185bde71afb21ebc132e1a4f/f: closing flushed file at 1732628177661 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53db3fad: reopening flushed file at 1732628178142 (+481 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 1d27e55c185bde71afb21ebc132e1a4f in 561ms, sequenceid=103, compaction requested=false at 1732628178163 (+21 ms)Writing region close event to WAL at 1732628178165 (+2 ms)Running coprocessor post-close hooks at 1732628178172 (+7 ms)Closed at 1732628178174 (+2 ms) 2024-11-26T13:36:18,175 WARN [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 1d27e55c185bde71afb21ebc132e1a4f to self. 2024-11-26T13:36:18,178 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,179 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=CLOSED 2024-11-26T13:36:18,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:18,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-26T13:36:18,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 in 748 msec 2024-11-26T13:36:18,187 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE; state=CLOSED, location=256175a329f8,38807,1732628159830; forceNewPlan=false, retain=true 2024-11-26T13:36:18,338 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:18,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE because future has completed 2024-11-26T13:36:18,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:18,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17326281684491 2024-11-26T13:36:18,372 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17326281684491 Metrics about Tables on a single HBase RegionServer 2024-11-26T13:36:18,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17326281684493 2024-11-26T13:36:18,373 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17326281684493 Metrics about Tables on a single HBase RegionServer 2024-11-26T13:36:18,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17326281684492 2024-11-26T13:36:18,374 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17326281684492 Metrics about Tables on a single HBase RegionServer 2024-11-26T13:36:18,500 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:18,500 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d27e55c185bde71afb21ebc132e1a4f, NAME => 'test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:18,501 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:18,501 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732628168449 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,501 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:18,501 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,501 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,503 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,505 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d27e55c185bde71afb21ebc132e1a4f columnFamilyName f 2024-11-26T13:36:18,505 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:18,514 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 2024-11-26T13:36:18,515 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(327): Store=1d27e55c185bde71afb21ebc132e1a4f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:18,515 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,517 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d27e55c185bde71afb21ebc132e1a4f columnFamilyName f3 2024-11-26T13:36:18,517 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:18,517 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(327): Store=1d27e55c185bde71afb21ebc132e1a4f/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:18,519 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,526 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d27e55c185bde71afb21ebc132e1a4f columnFamilyName mob 2024-11-26T13:36:18,530 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:18,531 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(327): Store=1d27e55c185bde71afb21ebc132e1a4f/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:18,531 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,532 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,534 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,536 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,536 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,537 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732628168449 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T13:36:18,539 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,541 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 1d27e55c185bde71afb21ebc132e1a4f; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11963692160, jitterRate=0.11420565843582153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T13:36:18,541 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:18,542 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 1d27e55c185bde71afb21ebc132e1a4f: Running coprocessor pre-open hook at 1732628178501Writing region info on filesystem at 1732628178501Initializing all the Stores at 1732628178503 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628178503Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628178503Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628178503Cleaning up temporary data from old regions at 1732628178536 (+33 ms)Running coprocessor post-open hooks at 1732628178541 (+5 ms)Region opened successfully at 1732628178542 (+1 ms) 2024-11-26T13:36:18,543 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., pid=24, masterSystemTime=1732628178494 2024-11-26T13:36:18,547 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:18,547 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:18,547 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=OPEN, openSeqNum=107, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:18,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:18,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-26T13:36:18,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 in 210 msec 2024-11-26T13:36:18,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-26T13:36:18,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE in 1.1250 sec 2024-11-26T13:36:18,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-26T13:36:18,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732628168449 in 1.1390 sec 2024-11-26T13:36:18,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732628168449 in 1.5760 sec 2024-11-26T13:36:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-26T13:36:18,569 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732628168449 completed 2024-11-26T13:36:18,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d398a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:18,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:18,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:18,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:18,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:18,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:18,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5cb0d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:18,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:18,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:18,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:18,574 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:18,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674886ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:18,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:18,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:18,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:18,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:18,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732628168449', row='row-test-1732628168449-3-0', locateType=CURRENT is [region=test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., hostname=256175a329f8,38807,1732628159830, seqNum=107] 2024-11-26T13:36:18,621 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T13:36:18,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 256175a329f8,40825,1732628157903 2024-11-26T13:36:18,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@306fa0ba 2024-11-26T13:36:18,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T13:36:18,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T13:36:18,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-11-26T13:36:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-26T13:36:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T13:36:18,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 10 msec 2024-11-26T13:36:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T13:36:18,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-26T13:36:18,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-26T13:36:18,774 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T13:36:18,774 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:18,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-26T13:36:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T13:36:18,775 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T13:36:18,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741849_1025 (size=635) 2024-11-26T13:36:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T13:36:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T13:36:19,187 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => be93584de20547dc73771fcf1a1b4605, NAME => 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:19,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741850_1026 (size=40) 2024-11-26T13:36:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T13:36:19,597 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:19,597 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing be93584de20547dc73771fcf1a1b4605, disabling compactions & flushes 2024-11-26T13:36:19,597 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:19,597 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:19,597 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. after waiting 0 ms 2024-11-26T13:36:19,598 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:19,598 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:19,598 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for be93584de20547dc73771fcf1a1b4605: Waiting for close lock at 1732628179597Disabling compacts and flushes for region at 1732628179597Disabling writes for close at 1732628179597Writing region close event to WAL at 1732628179598 (+1 ms)Closed at 1732628179598 2024-11-26T13:36:19,600 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T13:36:19,600 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732628179600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732628179600"}]},"ts":"1732628179600"} 2024-11-26T13:36:19,603 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T13:36:19,604 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T13:36:19,604 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628179604"}]},"ts":"1732628179604"} 2024-11-26T13:36:19,606 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-26T13:36:19,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN}] 2024-11-26T13:36:19,608 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN 2024-11-26T13:36:19,609 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN; state=OFFLINE, location=256175a329f8,38807,1732628159830; forceNewPlan=false, retain=false 2024-11-26T13:36:19,760 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=be93584de20547dc73771fcf1a1b4605, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:19,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN because future has completed 2024-11-26T13:36:19,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T13:36:19,922 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:19,923 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => be93584de20547dc73771fcf1a1b4605, NAME => 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:19,923 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:19,923 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,923 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:19,923 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,924 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,925 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,927 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be93584de20547dc73771fcf1a1b4605 columnFamilyName meta 2024-11-26T13:36:19,928 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:19,928 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.HStore(327): Store=be93584de20547dc73771fcf1a1b4605/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:19,928 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,930 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be93584de20547dc73771fcf1a1b4605 columnFamilyName session 2024-11-26T13:36:19,931 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:19,931 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.HStore(327): Store=be93584de20547dc73771fcf1a1b4605/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:19,932 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,933 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,933 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,935 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,935 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,936 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T13:36:19,938 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,941 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T13:36:19,942 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened be93584de20547dc73771fcf1a1b4605; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10076709920, jitterRate=-0.061533257365226746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T13:36:19,942 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:19,943 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for be93584de20547dc73771fcf1a1b4605: Running coprocessor pre-open hook at 1732628179924Writing region info on filesystem at 1732628179924Initializing all the Stores at 1732628179925 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628179925Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628179925Cleaning up temporary data from old regions at 1732628179935 (+10 ms)Running coprocessor post-open hooks at 1732628179942 (+7 ms)Region opened successfully at 1732628179943 (+1 ms) 2024-11-26T13:36:19,944 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., pid=28, masterSystemTime=1732628179917 2024-11-26T13:36:19,947 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:19,947 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:19,948 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=be93584de20547dc73771fcf1a1b4605, regionState=OPEN, openSeqNum=2, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:19,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:19,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-26T13:36:19,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830 in 187 msec 2024-11-26T13:36:19,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-26T13:36:19,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN in 347 msec 2024-11-26T13:36:19,959 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T13:36:19,959 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628179959"}]},"ts":"1732628179959"} 2024-11-26T13:36:19,962 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-26T13:36:19,963 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T13:36:19,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.1920 sec 2024-11-26T13:36:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-26T13:36:20,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-26T13:36:20,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-26T13:36:20,947 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T13:36:20,947 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:20,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-26T13:36:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T13:36:20,948 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T13:36:20,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741851_1027 (size=640) 2024-11-26T13:36:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T13:36:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T13:36:21,360 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 739931d97a34c012b15574f47d4b4161, NAME => 'backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:21,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741852_1028 (size=45) 2024-11-26T13:36:21,368 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:21,369 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing 739931d97a34c012b15574f47d4b4161, disabling compactions & flushes 2024-11-26T13:36:21,369 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:21,369 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:21,369 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. after waiting 0 ms 2024-11-26T13:36:21,369 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:21,369 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:21,369 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for 739931d97a34c012b15574f47d4b4161: Waiting for close lock at 1732628181369Disabling compacts and flushes for region at 1732628181369Disabling writes for close at 1732628181369Writing region close event to WAL at 1732628181369Closed at 1732628181369 2024-11-26T13:36:21,371 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T13:36:21,371 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732628181371"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732628181371"}]},"ts":"1732628181371"} 2024-11-26T13:36:21,374 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T13:36:21,375 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T13:36:21,375 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628181375"}]},"ts":"1732628181375"} 2024-11-26T13:36:21,377 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-26T13:36:21,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=739931d97a34c012b15574f47d4b4161, ASSIGN}] 2024-11-26T13:36:21,379 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=739931d97a34c012b15574f47d4b4161, ASSIGN 2024-11-26T13:36:21,380 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=739931d97a34c012b15574f47d4b4161, ASSIGN; state=OFFLINE, location=256175a329f8,38807,1732628159830; forceNewPlan=false, retain=false 2024-11-26T13:36:21,439 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-26T13:36:21,465 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17326281684493' 2024-11-26T13:36:21,465 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-26T13:36:21,530 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=739931d97a34c012b15574f47d4b4161, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:21,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=739931d97a34c012b15574f47d4b4161, ASSIGN because future has completed 2024-11-26T13:36:21,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 739931d97a34c012b15574f47d4b4161, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T13:36:21,691 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:21,691 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 739931d97a34c012b15574f47d4b4161, NAME => 'backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:21,692 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:21,692 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,692 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:21,692 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,692 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,694 INFO [StoreOpener-739931d97a34c012b15574f47d4b4161-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,696 INFO [StoreOpener-739931d97a34c012b15574f47d4b4161-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739931d97a34c012b15574f47d4b4161 columnFamilyName meta 2024-11-26T13:36:21,696 DEBUG [StoreOpener-739931d97a34c012b15574f47d4b4161-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:21,697 INFO [StoreOpener-739931d97a34c012b15574f47d4b4161-1 {}] regionserver.HStore(327): Store=739931d97a34c012b15574f47d4b4161/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:21,697 INFO [StoreOpener-739931d97a34c012b15574f47d4b4161-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,699 INFO [StoreOpener-739931d97a34c012b15574f47d4b4161-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739931d97a34c012b15574f47d4b4161 columnFamilyName session 2024-11-26T13:36:21,699 DEBUG [StoreOpener-739931d97a34c012b15574f47d4b4161-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:21,700 INFO [StoreOpener-739931d97a34c012b15574f47d4b4161-1 {}] regionserver.HStore(327): Store=739931d97a34c012b15574f47d4b4161/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:21,700 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,701 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system_bulk/739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,701 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system_bulk/739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,702 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,702 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,703 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T13:36:21,705 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,708 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system_bulk/739931d97a34c012b15574f47d4b4161/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T13:36:21,708 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 739931d97a34c012b15574f47d4b4161; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11284546560, jitterRate=0.05095529556274414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T13:36:21,708 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:21,709 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 739931d97a34c012b15574f47d4b4161: Running coprocessor pre-open hook at 1732628181692Writing region info on filesystem at 1732628181692Initializing all the Stores at 1732628181694 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628181694Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628181694Cleaning up temporary data from old regions at 1732628181702 (+8 ms)Running coprocessor post-open hooks at 1732628181708 (+6 ms)Region opened successfully at 1732628181709 (+1 ms) 2024-11-26T13:36:21,711 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161., pid=31, masterSystemTime=1732628181686 2024-11-26T13:36:21,714 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:21,714 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:21,715 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=739931d97a34c012b15574f47d4b4161, regionState=OPEN, openSeqNum=2, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:21,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 739931d97a34c012b15574f47d4b4161, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:21,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-26T13:36:21,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 739931d97a34c012b15574f47d4b4161, server=256175a329f8,38807,1732628159830 in 185 msec 2024-11-26T13:36:21,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-26T13:36:21,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=739931d97a34c012b15574f47d4b4161, ASSIGN in 344 msec 2024-11-26T13:36:21,725 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T13:36:21,725 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628181725"}]},"ts":"1732628181725"} 2024-11-26T13:36:21,728 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-26T13:36:21,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T13:36:21,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 785 msec 2024-11-26T13:36:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T13:36:22,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-26T13:36:22,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:22,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:22,101 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-26T13:36:22,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., hostname=256175a329f8,38807,1732628159830, seqNum=2] 2024-11-26T13:36:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-26T13:36:22,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:36:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628182127 (current time:1732628182127). 2024-11-26T13:36:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:36:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-26T13:36:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d3b05a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:22,130 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:22,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:22,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:22,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263ad1cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:22,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:22,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:22,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:22,132 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:22,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37159aba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:22,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:22,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:22,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:22,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:36:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:22,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:36:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:36:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-26T13:36:22,152 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:36:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T13:36:22,158 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:36:22,175 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:36:22,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741853_1029 (size=52) 2024-11-26T13:36:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T13:36:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T13:36:22,586 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:36:22,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605}] 2024-11-26T13:36:22,593 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:22,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-26T13:36:22,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:22,752 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing be93584de20547dc73771fcf1a1b4605 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-26T13:36:22,782 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/d39591ef2cef4c0299ebddb16931dca4 is 49, key is activesession:/session:c/1732628182116/Put/seqid=0 2024-11-26T13:36:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T13:36:22,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741854_1030 (size=5049) 2024-11-26T13:36:23,189 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/d39591ef2cef4c0299ebddb16931dca4 2024-11-26T13:36:23,200 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/d39591ef2cef4c0299ebddb16931dca4 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 2024-11-26T13:36:23,208 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4, entries=1, sequenceid=5, filesize=4.9 K 2024-11-26T13:36:23,209 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for be93584de20547dc73771fcf1a1b4605 in 457ms, sequenceid=5, compaction requested=false 2024-11-26T13:36:23,209 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for be93584de20547dc73771fcf1a1b4605: 2024-11-26T13:36:23,209 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. for snapshot_backup_system completed. 2024-11-26T13:36:23,210 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.' region-info for snapshot=snapshot_backup_system 2024-11-26T13:36:23,214 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:36:23,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-26T13:36:23,216 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4] hfiles 2024-11-26T13:36:23,218 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 for snapshot=snapshot_backup_system 2024-11-26T13:36:23,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741855_1031 (size=95) 2024-11-26T13:36:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T13:36:23,629 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:23,632 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-26T13:36:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-26T13:36:23,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:23,635 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:23,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-26T13:36:23,639 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:36:23,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 in 1.0480 sec 2024-11-26T13:36:23,641 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:36:23,642 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:36:23,642 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-26T13:36:23,645 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-26T13:36:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741856_1032 (size=731) 2024-11-26T13:36:24,066 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:36:24,079 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:36:24,080 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_backup_system 2024-11-26T13:36:24,082 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:36:24,083 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-26T13:36:24,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9370 sec 2024-11-26T13:36:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-26T13:36:24,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-26T13:36:24,305 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732628178606 started at 1732628184304. 2024-11-26T13:36:24,320 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732628178606 has been started. 2024-11-26T13:36:24,326 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-26T13:36:24,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-26T13:36:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-26T13:36:24,335 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-26T13:36:24,336 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-26T13:36:24,336 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-26T13:36:24,336 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-26T13:36:24,337 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,337 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:24,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-26T13:36:24,339 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,339 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-26T13:36:24,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:36:24,339 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,340 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-26T13:36:24,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:24,341 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,342 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-26T13:36:24,342 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:24,342 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-26T13:36:24,344 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-26T13:36:24,344 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-26T13:36:24,345 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-26T13:36:24,345 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-26T13:36:24,345 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-26T13:36:24,345 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-26T13:36:24,345 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '256175a329f8,38807,1732628159830' joining acquired barrier for procedure (rolllog) in zk 2024-11-26T13:36:24,348 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:24,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,348 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,348 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:24,348 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:24,348 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-26T13:36:24,348 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-26T13:36:24,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:24,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:24,350 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:24,350 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:24,351 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:24,351 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '256175a329f8,38807,1732628159830' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-26T13:36:24,352 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-26T13:36:24,352 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2881c08f[Count = 0] remaining members to acquire global barrier 2024-11-26T13:36:24,352 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:24,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:24,353 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:24,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:24,354 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,354 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-26T13:36:24,354 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-26T13:36:24,354 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-26T13:36:24,355 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 256175a329f8,38807,1732628159830 2024-11-26T13:36:24,361 INFO [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732628163201 highest: 1732628169762 on 256175a329f8,38807,1732628159830 2024-11-26T13:36:24,362 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.meta:.meta(num 1732628163597) roll requested 2024-11-26T13:36:24,376 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628184362.meta, exclude list is [], retry=0 2024-11-26T13:36:24,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:24,382 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628163597.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628184362.meta 2024-11-26T13:36:24,383 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:24,383 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628163597.meta is not closed yet, will try archiving it next time 2024-11-26T13:36:24,383 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1:(num 1732628169762) roll requested 2024-11-26T13:36:24,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741834_1010 (size=12055) 2024-11-26T13:36:24,399 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383, exclude list is [], retry=0 2024-11-26T13:36:24,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:24,405 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628169762 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 2024-11-26T13:36:24,406 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:24,406 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628169762 is not closed yet, will try archiving it next time 2024-11-26T13:36:24,406 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0:(num 1732628163201) roll requested 2024-11-26T13:36:24,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741840_1016 (size=4756) 2024-11-26T13:36:24,408 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628169762 not finished, retry = 0 2024-11-26T13:36:24,421 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406, exclude list is [], retry=0 2024-11-26T13:36:24,422 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-26T13:36:24,423 INFO [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732628184406 on 256175a329f8,38807,1732628159830 2024-11-26T13:36:24,423 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T13:36:24,423 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is 256175a329f8,40825,1732628157903 2024-11-26T13:36:24,423 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56d6a854 2024-11-26T13:36:24,423 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T13:36:24,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:24,425 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34031, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-26T13:36:24,426 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:24,426 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:24,427 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:24,427 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628163201 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 2024-11-26T13:36:24,427 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:24,427 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628163201 is not closed yet, will try archiving it next time 2024-11-26T13:36:24,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33663, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-26T13:36:24,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741833_1009 (size=1034) 2024-11-26T13:36:24,431 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628163201 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628163201 2024-11-26T13:36:24,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:24,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:24,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:36063/backupUT\x00', locateType=CURRENT is [region=backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., hostname=256175a329f8,38807,1732628159830, seqNum=2] 2024-11-26T13:36:24,453 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-26T13:36:24,453 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '256175a329f8,38807,1732628159830' in zk 2024-11-26T13:36:24,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,456 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-26T13:36:24,456 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-26T13:36:24,456 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,456 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:24,456 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-26T13:36:24,456 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:24,457 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:24,457 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:24,458 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:24,458 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:24,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:24,459 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:24,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:24,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '256175a329f8,38807,1732628159830': 2024-11-26T13:36:24,461 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-26T13:36:24,461 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-26T13:36:24,461 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-26T13:36:24,461 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-26T13:36:24,461 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '256175a329f8,38807,1732628159830' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-26T13:36:24,461 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-26T13:36:24,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-26T13:36:24,464 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,464 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,464 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,464 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:24,464 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:24,465 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-26T13:36:24,465 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:24,465 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:36:24,465 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,465 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,466 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:24,466 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:24,467 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:24,467 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:24,467 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:24,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:24,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:24,475 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:24,475 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:24,475 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-26T13:36:24,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-26T13:36:24,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-26T13:36:24,476 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-26T13:36:24,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:36:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-26T13:36:24,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } 2024-11-26T13:36:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628184491 (current time:1732628184491). 2024-11-26T13:36:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:36:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732628184490_default_test-1732628168449 VERSION not specified, setting to 2 2024-11-26T13:36:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6afc8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:24,493 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:24,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:24,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:24,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3ae86a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:24,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:24,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:24,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:24,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:24,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c87c1ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:24,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:24,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:24,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:24,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:36:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:24,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:36:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } 2024-11-26T13:36:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-26T13:36:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T13:36:24,507 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:36:24,508 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:36:24,512 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:36:24,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741860_1036 (size=84) 2024-11-26T13:36:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T13:36:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T13:36:24,920 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:36:24,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f}] 2024-11-26T13:36:24,921 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:25,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-26T13:36:25,075 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:25,075 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 1d27e55c185bde71afb21ebc132e1a4f 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-26T13:36:25,095 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f3/d47db011f3204697a9be35330503704b is 58, key is row-test-1732628168449-3-0/f3:q1/1732628178583/Put/seqid=0 2024-11-26T13:36:25,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741861_1037 (size=5387) 2024-11-26T13:36:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T13:36:25,502 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f3/d47db011f3204697a9be35330503704b 2024-11-26T13:36:25,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741862_1038 (size=4469) 2024-11-26T13:36:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T13:36:25,962 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/mob/31d81ec77ef04d11b069a5ab4c7d18c5 is 59, key is row-test-1732628168449-3-0/mob:q1/1732628178594/Put/seqid=0 2024-11-26T13:36:25,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741863_1039 (size=5484) 2024-11-26T13:36:26,372 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/mob/31d81ec77ef04d11b069a5ab4c7d18c5 2024-11-26T13:36:26,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f3/d47db011f3204697a9be35330503704b as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b 2024-11-26T13:36:26,393 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b, entries=6, sequenceid=121, filesize=5.3 K 2024-11-26T13:36:26,394 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/mob/31d81ec77ef04d11b069a5ab4c7d18c5 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 2024-11-26T13:36:26,403 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5, entries=6, sequenceid=121, filesize=5.4 K 2024-11-26T13:36:26,405 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 1d27e55c185bde71afb21ebc132e1a4f in 1329ms, sequenceid=121, compaction requested=false 2024-11-26T13:36:26,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 1d27e55c185bde71afb21ebc132e1a4f: 2024-11-26T13:36:26,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. for snapshot_1732628184490_default_test-1732628168449 completed. 2024-11-26T13:36:26,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.' region-info for snapshot=snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:26,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:36:26,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433] hfiles 2024-11-26T13:36:26,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 for snapshot=snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:26,406 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b] hfiles 2024-11-26T13:36:26,406 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b for snapshot=snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:26,406 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5] hfiles 2024-11-26T13:36:26,406 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 for snapshot=snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741864_1040 (size=185) 2024-11-26T13:36:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T13:36:26,813 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:26,814 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-26T13:36:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-26T13:36:26,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732628184490_default_test-1732628168449 on region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:26,815 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:26,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-26T13:36:26,819 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:36:26,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f in 1.8960 sec 2024-11-26T13:36:26,820 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:36:26,822 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-26T13:36:26,822 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-26T13:36:26,823 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:26,824 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-26T13:36:26,824 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:26,824 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-26T13:36:26,825 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:26,825 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-26T13:36:26,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741865_1041 (size=49) 2024-11-26T13:36:26,892 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T13:36:27,194 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-26T13:36:27,234 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:36:27,234 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:27,235 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:27,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741866_1042 (size=1146) 2024-11-26T13:36:27,466 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-26T13:36:27,650 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:36:27,659 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:36:27,660 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628184490_default_test-1732628168449 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:27,661 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:36:27,662 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-26T13:36:27,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732628184490_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } in 3.1580 sec 2024-11-26T13:36:28,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-26T13:36:28,372 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-26T13:36:28,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732628168449 2024-11-26T13:36:28,373 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1732628168449 Metrics about Tables on a single HBase RegionServer 2024-11-26T13:36:28,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-26T13:36:28,373 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-26T13:36:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-26T13:36:28,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732628168449 completed 2024-11-26T13:36:28,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } 2024-11-26T13:36:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628188675 (current time:1732628188675). 2024-11-26T13:36:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:36:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732628188674_ns2_test-17326281684491 VERSION not specified, setting to 2 2024-11-26T13:36:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef0982c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:28,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:28,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:28,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:28,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9b8102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:28,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:28,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:28,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:28,679 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:28,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73808c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:28,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:28,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:28,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:28,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:36:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:28,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:36:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } 2024-11-26T13:36:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-26T13:36:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T13:36:28,687 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:36:28,688 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:36:28,692 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:36:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741867_1043 (size=86) 2024-11-26T13:36:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T13:36:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T13:36:29,100 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:36:29,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d333b64a51b935fe3679e64e25b2142a}] 2024-11-26T13:36:29,101 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:29,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-26T13:36:29,254 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:29,254 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing d333b64a51b935fe3679e64e25b2142a 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-26T13:36:29,274 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/.tmp/f/fddff099324e4695856d43ec7e2b8550 is 37, key is row10/f:q1/1732628172457/Put/seqid=0 2024-11-26T13:36:29,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741868_1044 (size=8413) 2024-11-26T13:36:29,279 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/.tmp/f/fddff099324e4695856d43ec7e2b8550 2024-11-26T13:36:29,288 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/.tmp/f/fddff099324e4695856d43ec7e2b8550 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550 2024-11-26T13:36:29,295 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550, entries=99, sequenceid=103, filesize=8.2 K 2024-11-26T13:36:29,296 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for d333b64a51b935fe3679e64e25b2142a in 42ms, sequenceid=103, compaction requested=false 2024-11-26T13:36:29,297 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for d333b64a51b935fe3679e64e25b2142a: 2024-11-26T13:36:29,297 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. for snapshot_1732628188674_ns2_test-17326281684491 completed. 2024-11-26T13:36:29,297 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a.' region-info for snapshot=snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:29,297 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:36:29,297 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550] hfiles 2024-11-26T13:36:29,297 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550 for snapshot=snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:29,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741869_1045 (size=91) 2024-11-26T13:36:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T13:36:29,704 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:29,704 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-26T13:36:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-26T13:36:29,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732628188674_ns2_test-17326281684491 on region d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:29,705 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:29,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-26T13:36:29,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d333b64a51b935fe3679e64e25b2142a in 606 msec 2024-11-26T13:36:29,708 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:36:29,709 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:36:29,710 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:36:29,710 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:29,711 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:29,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741870_1046 (size=456) 2024-11-26T13:36:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T13:36:30,124 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:36:30,132 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:36:30,132 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628188674_ns2_test-17326281684491 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:30,134 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:36:30,134 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-26T13:36:30,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732628188674_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } in 1.4490 sec 2024-11-26T13:36:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T13:36:30,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17326281684491 completed 2024-11-26T13:36:30,834 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732628178606 2024-11-26T13:36:30,834 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-26T13:36:30,835 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-26T13:36:30,838 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732628178606_ns2:test-17326281684491 2024-11-26T13:36:30,838 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732628188674_ns2_test-17326281684491 to hdfs://localhost:36063/backupUT/backup_1732628178606/ns2/test-17326281684491/ 2024-11-26T13:36:30,844 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-26T13:36:30,880 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36063, inputRoot=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:30,880 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1216694942_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36063/backupUT/backup_1732628178606/ns2/test-17326281684491, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36063/backupUT/backup_1732628178606/ns2/test-17326281684491/.hbase-snapshot/.tmp/snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:30,883 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-26T13:36:30,889 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628188674_ns2_test-17326281684491 to hdfs://localhost:36063/backupUT/backup_1732628178606/ns2/test-17326281684491/.hbase-snapshot/.tmp/snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:30,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741872_1048 (size=456) 2024-11-26T13:36:30,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741871_1047 (size=86) 2024-11-26T13:36:31,309 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:31,310 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:31,310 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,336 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-13162028193455014997.jar 2024-11-26T13:36:32,336 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,337 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,337 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,338 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,338 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,338 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,339 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,339 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,339 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:32,340 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:36:32,340 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:36:32,340 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:36:32,341 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:36:32,341 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:36:32,341 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:36:32,342 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:36:32,342 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:36:32,342 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:36:32,343 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:36:32,343 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:36:32,346 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:36:32,346 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:36:32,346 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:36:32,347 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:36:32,347 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:36:32,347 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:36:32,347 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:36:32,418 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:36:32,452 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732628188674_ns2_test-17326281684491' hfile list 2024-11-26T13:36:32,459 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-11-26T13:36:32,687 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local508529126_0001_6243d038-f292-448f-ad29-c658977c038a/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-26T13:36:32,687 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local508529126_0001_6243d038-f292-448f-ad29-c658977c038a/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-26T13:36:32,834 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-26T13:36:32,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741873_1049 (size=8413) 2024-11-26T13:36:33,343 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550 output=hdfs://localhost:36063/backupUT/backup_1732628178606/ns2/test-17326281684491/archive/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550 2024-11-26T13:36:33,345 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.016M/sec 2024-11-26T13:36:33,755 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-26T13:36:33,757 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-26T13:36:33,764 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:33,764 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732628188674_ns2_test-17326281684491 finished. 2024-11-26T13:36:33,764 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732628178606_test-1732628168449 2024-11-26T13:36:33,764 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732628184490_default_test-1732628168449 to hdfs://localhost:36063/backupUT/backup_1732628178606/default/test-1732628168449/ 2024-11-26T13:36:33,764 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-26T13:36:33,796 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36063, inputRoot=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:36:33,796 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1216694942_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36063/backupUT/backup_1732628178606/default/test-1732628168449, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36063/backupUT/backup_1732628178606/default/test-1732628168449/.hbase-snapshot/.tmp/snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:33,799 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-26T13:36:33,806 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628184490_default_test-1732628168449 to hdfs://localhost:36063/backupUT/backup_1732628178606/default/test-1732628168449/.hbase-snapshot/.tmp/snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:33,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741874_1050 (size=84) 2024-11-26T13:36:33,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741875_1051 (size=1146) 2024-11-26T13:36:34,221 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:34,221 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:34,221 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,221 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-14368613478789734019.jar 2024-11-26T13:36:35,221 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,222 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,222 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,222 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,223 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,223 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,223 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,223 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,224 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:35,224 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:36:35,224 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:36:35,224 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:36:35,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:36:35,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:36:35,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:36:35,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:36:35,226 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:36:35,226 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:36:35,226 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:36:35,226 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:36:35,227 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:36:35,227 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:36:35,227 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:36:35,228 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:36:35,228 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:36:35,228 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:36:35,228 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:36:35,272 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:36:35,300 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732628184490_default_test-1732628168449' hfile list 2024-11-26T13:36:35,303 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-11-26T13:36:35,395 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1038872043_0002_46c2496d-9d08-4481-99c2-f2442e49f80e/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-26T13:36:35,396 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1038872043_0002_46c2496d-9d08-4481-99c2-f2442e49f80e/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-26T13:36:35,475 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-26T13:36:35,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741876_1052 (size=8413) 2024-11-26T13:36:35,922 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 output=hdfs://localhost:36063/backupUT/backup_1732628178606/default/test-1732628168449/archive/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 2024-11-26T13:36:35,923 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-26T13:36:35,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741877_1053 (size=5484) 2024-11-26T13:36:36,397 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 output=hdfs://localhost:36063/backupUT/backup_1732628178606/default/test-1732628168449/archive/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 2024-11-26T13:36:36,398 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.011M/sec 2024-11-26T13:36:36,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741878_1054 (size=5387) 2024-11-26T13:36:36,870 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b output=hdfs://localhost:36063/backupUT/backup_1732628178606/default/test-1732628168449/archive/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b 2024-11-26T13:36:36,870 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-11-26T13:36:37,442 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-26T13:36:37,443 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-26T13:36:37,451 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:37,452 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732628184490_default_test-1732628168449 finished. 2024-11-26T13:36:37,452 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17326281684491 2024-11-26T13:36:37,452 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732628168449 2024-11-26T13:36:37,506 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732628178606 2024-11-26T13:36:37,506 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-26T13:36:37,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741879_1055 (size=130) 2024-11-26T13:36:37,916 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36063/backupUT/backup_1732628178606/.backup.manifest 2024-11-26T13:36:37,918 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732628178606 finished: type=FULL,tablelist=ns2:test-17326281684491;test-1732628168449,targetRootDir=hdfs://localhost:36063/backupUT,startts=1732628184304,completets=1732628197504,bytescopied=0 2024-11-26T13:36:37,918 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-26T13:36:37,919 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:37,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732628188674_ns2_test-17326281684491" type: DISABLED 2024-11-26T13:36:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732628188674_ns2_test-17326281684491 2024-11-26T13:36:37,927 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732628188674_ns2_test-17326281684491 for backup backup_1732628178606 succeeded. 2024-11-26T13:36:37,927 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:37,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732628184490_default_test-1732628168449" type: DISABLED 2024-11-26T13:36:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732628184490_default_test-1732628168449 2024-11-26T13:36:37,932 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732628184490_default_test-1732628168449 for backup backup_1732628178606 succeeded. 2024-11-26T13:36:37,933 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-26T13:36:37,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-26T13:36:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-26T13:36:37,947 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-26T13:36:37,953 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-26T13:36:37,970 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732628178606 completed. 2024-11-26T13:36:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1732628178606', locateType=CURRENT is [region=backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., hostname=256175a329f8,38807,1732628159830, seqNum=2] 2024-11-26T13:36:38,141 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1732628168449 2024-11-26T13:36:38,158 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1732628168449 2024-11-26T13:36:38,326 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1732628168449 to Mob enabled CF 2024-11-26T13:36:38,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17326281684491', row='row-t20', locateType=CURRENT is [region=ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a., hostname=256175a329f8,38807,1732628159830, seqNum=2] 2024-11-26T13:36:38,356 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17326281684491 2024-11-26T13:36:38,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:38,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] assignment.SplitTableRegionProcedure(210): test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. splitKey isn't explicitly specified, will try to find a best split key from RS 256175a329f8,38807,1732628159830 2024-11-26T13:36:38,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 because midkey is the same as first or last row 2024-11-26T13:36:38,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b because midkey is the same as first or last row 2024-11-26T13:36:38,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 because midkey is the same as first or last row 2024-11-26T13:36:38,368 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegion(2902): Flushing 1d27e55c185bde71afb21ebc132e1a4f 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-26T13:36:38,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f/497ea1f8599b476c8770fb402b3b1cd8 is 59, key is row-test-1732628168449-1-10/f:q1/1732628198010/Put/seqid=0 2024-11-26T13:36:38,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741880_1056 (size=10796) 2024-11-26T13:36:38,795 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f/497ea1f8599b476c8770fb402b3b1cd8 2024-11-26T13:36:38,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f is 63, key is row-test-1732628168449-3-100/mob:q1/1732628198309/Put/seqid=0 2024-11-26T13:36:38,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741881_1057 (size=5591) 2024-11-26T13:36:39,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:39,225 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/mobdir/data/default/test-1732628168449/d2b2517e3714bd246bf40fd99916840e/mob/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:39,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/mob/21d654267d294ec594bc4a9cd1f8f382, store: [table=test-1732628168449 family=mob region=1d27e55c185bde71afb21ebc132e1a4f] 2024-11-26T13:36:39,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/mob/21d654267d294ec594bc4a9cd1f8f382 is 192, key is row-test-1732628168449-3-100/mob:q1/1732628198309/Put/seqid=0 2024-11-26T13:36:39,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741882_1058 (size=13760) 2024-11-26T13:36:39,633 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/mob/21d654267d294ec594bc4a9cd1f8f382 2024-11-26T13:36:39,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f/497ea1f8599b476c8770fb402b3b1cd8 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/497ea1f8599b476c8770fb402b3b1cd8 2024-11-26T13:36:39,648 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/497ea1f8599b476c8770fb402b3b1cd8, entries=99, sequenceid=334, filesize=10.5 K 2024-11-26T13:36:39,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/mob/21d654267d294ec594bc4a9cd1f8f382 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/21d654267d294ec594bc4a9cd1f8f382 2024-11-26T13:36:39,656 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/21d654267d294ec594bc4a9cd1f8f382, entries=111, sequenceid=334, filesize=13.4 K 2024-11-26T13:36:39,658 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 1d27e55c185bde71afb21ebc132e1a4f in 1290ms, sequenceid=334, compaction requested=false 2024-11-26T13:36:39,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegion(2603): Flush status journal for 1d27e55c185bde71afb21ebc132e1a4f: 2024-11-26T13:36:39,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/497ea1f8599b476c8770fb402b3b1cd8 because midkey is the same as first or last row 2024-11-26T13:36:39,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b because midkey is the same as first or last row 2024-11-26T13:36:39,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/21d654267d294ec594bc4a9cd1f8f382 because midkey is the same as first or last row 2024-11-26T13:36:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=256175a329f8,38807,1732628159830 2024-11-26T13:36:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T13:36:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.CallRunner(138): callId: 302 service: MasterService methodName: SplitRegion size: 100 connection: 172.17.0.2:39002 deadline: 1732628258362, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-26T13:36:39,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1732628168449 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:39,666 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:39,667 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T13:36:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732628168449,, stopping at row=test-1732628168449 ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,678 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732628168449,, stopping at row=test-1732628168449 ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,687 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1330 2024-11-26T13:36:39,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,723 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-26T13:36:39,727 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732628199687 2024-11-26T13:36:39,735 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732628178606} 2024-11-26T13:36:39,735 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-26T13:36:39,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:39,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:36:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628199782 (current time:1732628199782). 2024-11-26T13:36:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:36:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-26T13:36:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ba46d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:39,784 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:39,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:39,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:39,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb86956, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:39,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:39,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:39,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:39,787 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:39,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a68c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:39,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:39,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:39,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:39,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:36:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:39,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:36:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:36:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-26T13:36:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-26T13:36:39,795 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:36:39,796 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:36:39,798 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:36:39,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741883_1059 (size=52) 2024-11-26T13:36:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-26T13:36:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-26T13:36:40,205 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:36:40,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605}] 2024-11-26T13:36:40,207 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:40,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-26T13:36:40,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:40,359 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing be93584de20547dc73771fcf1a1b4605 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-26T13:36:40,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/ef83da5ee1434703a7a27938ed98d745 is 158, key is trslm:hdfs://localhost:36063/backupUT\x00ns2:test-17326281684491/meta:log-roll-map/1732628197491/Put/seqid=0 2024-11-26T13:36:40,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741884_1060 (size=5914) 2024-11-26T13:36:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-26T13:36:40,781 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/ef83da5ee1434703a7a27938ed98d745 2024-11-26T13:36:40,791 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/87d571180ca341c4a22fd473d47010cf is 433, key is session:backup_1732628178606/session:context/1732628197951/Put/seqid=0 2024-11-26T13:36:40,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741885_1061 (size=5535) 2024-11-26T13:36:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-26T13:36:41,196 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/87d571180ca341c4a22fd473d47010cf 2024-11-26T13:36:41,204 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/ef83da5ee1434703a7a27938ed98d745 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745 2024-11-26T13:36:41,210 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745, entries=6, sequenceid=17, filesize=5.8 K 2024-11-26T13:36:41,211 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/87d571180ca341c4a22fd473d47010cf as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf 2024-11-26T13:36:41,218 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf, entries=2, sequenceid=17, filesize=5.4 K 2024-11-26T13:36:41,219 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for be93584de20547dc73771fcf1a1b4605 in 860ms, sequenceid=17, compaction requested=false 2024-11-26T13:36:41,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for be93584de20547dc73771fcf1a1b4605: 2024-11-26T13:36:41,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. for snapshot_backup_system completed. 2024-11-26T13:36:41,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.' region-info for snapshot=snapshot_backup_system 2024-11-26T13:36:41,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:36:41,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745] hfiles 2024-11-26T13:36:41,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745 for snapshot=snapshot_backup_system 2024-11-26T13:36:41,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf] hfiles 2024-11-26T13:36:41,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 for snapshot=snapshot_backup_system 2024-11-26T13:36:41,220 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf for snapshot=snapshot_backup_system 2024-11-26T13:36:41,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741886_1062 (size=173) 2024-11-26T13:36:41,625 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:41,626 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-26T13:36:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-26T13:36:41,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:41,627 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:41,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-26T13:36:41,629 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:36:41,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 in 1.4220 sec 2024-11-26T13:36:41,630 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:36:41,631 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:36:41,631 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-26T13:36:41,632 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-26T13:36:41,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741887_1063 (size=810) 2024-11-26T13:36:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-26T13:36:42,043 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:36:42,052 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:36:42,052 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_backup_system 2024-11-26T13:36:42,054 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:36:42,054 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-26T13:36:42,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2620 sec 2024-11-26T13:36:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-26T13:36:43,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-26T13:36:43,954 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732628199687 started at 1732628203953. 2024-11-26T13:36:43,956 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732628199687 has been started. 2024-11-26T13:36:43,957 DEBUG [master/256175a329f8:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732628168449 2024-11-26T13:36:43,962 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [test-1732628168449, ns2:test-17326281684491] 2024-11-26T13:36:43,966 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732628169762for backupID backup_1732628199687 2024-11-26T13:36:43,966 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-26T13:36:43,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-26T13:36:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-26T13:36:43,967 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-26T13:36:43,967 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-26T13:36:43,967 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-26T13:36:43,967 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-26T13:36:43,968 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:43,968 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:43,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741888_1064 (size=950) 2024-11-26T13:36:43,970 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:43,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-26T13:36:43,970 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-26T13:36:43,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:36:43,970 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:43,970 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-26T13:36:43,971 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:43,972 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:43,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-26T13:36:43,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:43,972 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-26T13:36:43,973 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-26T13:36:43,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-26T13:36:43,973 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-26T13:36:43,973 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-26T13:36:43,973 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-26T13:36:43,973 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-26T13:36:43,973 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '256175a329f8,38807,1732628159830' joining acquired barrier for procedure (rolllog) in zk 2024-11-26T13:36:43,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:43,975 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:43,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:43,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:43,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:43,975 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-26T13:36:43,975 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-26T13:36:43,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:43,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:43,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:43,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:43,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:43,978 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '256175a329f8,38807,1732628159830' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-26T13:36:43,978 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1789dd8f[Count = 0] remaining members to acquire global barrier 2024-11-26T13:36:43,978 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-26T13:36:43,978 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:43,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:43,980 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:43,980 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:43,980 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-26T13:36:43,981 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:43,981 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-26T13:36:43,981 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-26T13:36:43,981 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 256175a329f8,38807,1732628159830 2024-11-26T13:36:43,981 INFO [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732628184406 highest: 1732628184406 on 256175a329f8,38807,1732628159830 2024-11-26T13:36:43,981 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.meta:.meta(num 1732628184362) roll requested 2024-11-26T13:36:43,995 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628203981.meta, exclude list is [], retry=0 2024-11-26T13:36:43,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:44,001 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628184362.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628203981.meta 2024-11-26T13:36:44,001 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:44,001 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628184362.meta is not closed yet, will try archiving it next time 2024-11-26T13:36:44,001 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1:(num 1732628184383) roll requested 2024-11-26T13:36:44,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741857_1033 (size=93) 2024-11-26T13:36:44,003 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628184362.meta to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.meta.1732628184362.meta 2024-11-26T13:36:44,015 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628204001, exclude list is [], retry=0 2024-11-26T13:36:44,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:44,020 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628204001 2024-11-26T13:36:44,020 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:44,020 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 is not closed yet, will try archiving it next time 2024-11-26T13:36:44,020 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0:(num 1732628184406) roll requested 2024-11-26T13:36:44,021 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628169762 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628169762 2024-11-26T13:36:44,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741858_1034 (size=30490) 2024-11-26T13:36:44,022 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 not finished, retry = 0 2024-11-26T13:36:44,034 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628204020, exclude list is [], retry=0 2024-11-26T13:36:44,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:44,039 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628204020 2024-11-26T13:36:44,039 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:44,039 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 is not closed yet, will try archiving it next time 2024-11-26T13:36:44,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741859_1035 (size=1198) 2024-11-26T13:36:44,041 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 not finished, retry = 0 2024-11-26T13:36:44,042 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-26T13:36:44,042 INFO [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732628204020 on 256175a329f8,38807,1732628159830 2024-11-26T13:36:44,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:44,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:44,062 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-26T13:36:44,062 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '256175a329f8,38807,1732628159830' in zk 2024-11-26T13:36:44,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:44,064 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-26T13:36:44,064 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-26T13:36:44,064 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-26T13:36:44,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:44,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:44,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:44,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:44,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:44,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:44,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:44,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:44,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:44,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:44,068 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '256175a329f8,38807,1732628159830': 2024-11-26T13:36:44,068 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-26T13:36:44,068 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '256175a329f8,38807,1732628159830' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-26T13:36:44,068 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-26T13:36:44,068 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-26T13:36:44,068 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-26T13:36:44,068 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-26T13:36:44,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:44,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:44,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-26T13:36:44,070 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:44,070 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:44,070 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:44,070 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:44,070 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:44,071 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-26T13:36:44,071 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:36:44,071 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:44,071 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:44,071 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:44,071 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:44,072 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:44,072 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:44,073 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:44,073 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:44,073 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:44,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:44,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:44,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:44,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-26T13:36:44,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:44,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:44,076 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-26T13:36:44,077 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:36:44,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:44,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-26T13:36:44,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:44,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:44,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:44,077 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-26T13:36:44,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-26T13:36:44,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-26T13:36:44,077 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-26T13:36:44,077 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:36:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-26T13:36:44,086 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {256175a329f8:38807=1732628169762} newestTimestamps: {256175a329f8:38807=1732628184406} 2024-11-26T13:36:44,088 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 256175a329f8%2C38807%2C1732628159830.meta.1732628163597.meta 2024-11-26T13:36:44,088 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 256175a329f8%2C38807%2C1732628159830.meta.1732628203981.meta 2024-11-26T13:36:44,088 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 2024-11-26T13:36:44,088 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628204020 2024-11-26T13:36:44,088 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 2024-11-26T13:36:44,088 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628204001 2024-11-26T13:36:44,089 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.meta.1732628184362.meta 2024-11-26T13:36:44,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741892_1068 (size=364) 2024-11-26T13:36:44,372 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/.tabledesc/.tableinfo.0000000002.908 2024-11-26T13:36:44,372 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/.tabledesc/.tableinfo.0000000003.950 2024-11-26T13:36:44,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732628168449 in 418 msec 2024-11-26T13:36:44,501 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17326281684491 target: hdfs://localhost:36063/backupUT/backup_1732628199687/ns2/test-17326281684491 descriptor: 'ns2:test-17326281684491', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:44,501 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-26T13:36:44,505 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17326281684491 2024-11-26T13:36:44,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741893_1069 (size=50) 2024-11-26T13:36:44,911 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17326281684491 2024-11-26T13:36:44,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741894_1070 (size=950) 2024-11-26T13:36:45,324 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732628168449 target: hdfs://localhost:36063/backupUT/backup_1732628199687/default/test-1732628168449 descriptor: 'test-1732628168449', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:45,324 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-26T13:36:45,327 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732628168449 2024-11-26T13:36:45,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741895_1071 (size=53) 2024-11-26T13:36:45,734 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732628168449 2024-11-26T13:36:45,745 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:36063/backupUT/.tmp/backup_1732628199687 from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406;hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 2024-11-26T13:36:45,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13f6e346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:45,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:45,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:45,748 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:45,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:45,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:45,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab25566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:45,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:45,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:45,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:45,750 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:45,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7e2a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:45,751 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T13:36:45,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 256175a329f8,40825,1732628157903 2024-11-26T13:36:45,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@257a4e74 2024-11-26T13:36:45,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T13:36:45,753 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T13:36:45,759 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-26T13:36:45,759 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732628168449 2024-11-26T13:36:45,759 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732628168449,, stopping at row=test-1732628168449 ,, for max=2147483647 with caching=100 2024-11-26T13:36:45,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:45,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:45,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:45,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:45,764 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732628168449: test-1732628168449; 2024-11-26T13:36:45,765 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17326281684491 2024-11-26T13:36:45,765 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17326281684491,, stopping at row=ns2:test-17326281684491 ,, for max=2147483647 with caching=100 2024-11-26T13:36:45,767 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17326281684491: ns2:test-17326281684491; 2024-11-26T13:36:45,767 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-26T13:36:45,767 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/hbase-staging/partitions_5350ee71-749f-43d2-b106-9f72b82e06da 2024-11-26T13:36:45,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741896_1072 (size=192) 2024-11-26T13:36:46,193 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:46,193 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:46,194 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,187 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-11496909002891305136.jar 2024-11-26T13:36:47,187 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,187 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,188 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,188 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,188 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,188 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,189 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,189 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,189 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,190 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:36:47,190 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:36:47,190 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:36:47,190 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:36:47,190 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:36:47,191 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:36:47,191 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:36:47,191 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:36:47,191 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:36:47,192 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:36:47,192 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:36:47,193 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,194 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,194 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,194 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,194 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,195 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:36:47,195 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:36:47,195 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: default:test-1732628168449,ns2:test-17326281684491 2024-11-26T13:36:47,195 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T13:36:47,195 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:36:47,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:47,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:47,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:47,196 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:36:48,183 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-8107554499107418208.jar 2024-11-26T13:36:48,197 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:36:48,224 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 for WAL files 2024-11-26T13:36:48,225 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 2024-11-26T13:36:48,225 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 for WAL files 2024-11-26T13:36:48,226 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 2024-11-26T13:36:48,328 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local239838184_0003_1c341961-b292-4f2f-a085-8875b42fd1d1/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-26T13:36:48,328 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local239838184_0003_1c341961-b292-4f2f-a085-8875b42fd1d1/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-26T13:36:48,429 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 for hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 (-9223372036854775808:9223372036854775807) length:30490 2024-11-26T13:36:48,451 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-26T13:36:48,451 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-26T13:36:48,502 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 for hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 (-9223372036854775808:9223372036854775807) length:1198 2024-11-26T13:36:48,506 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-26T13:36:48,506 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-26T13:36:48,597 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0477cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:48,597 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:48,597 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:48,598 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:48,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:48,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:48,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d7b9ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:48,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:48,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:48,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:48,601 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:48,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a75b33b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:48,602 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:48,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:48,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:48,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:48,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17326281684491', row='row-t20', locateType=CURRENT is [region=ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a., hostname=256175a329f8,38807,1732628159830, seqNum=2] 2024-11-26T13:36:48,607 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-26T13:36:48,608 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:36:48,608 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:48,608 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:48,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:48,608 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-26T13:36:48,608 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 256175a329f8 2024-11-26T13:36:48,614 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/backupUT/.tmp/backup_1732628199687/_temporary/0/_temporary/attempt_local239838184_0003_r_000000_0/ns2/test-17326281684491/f/4eb1e6c0a8f7438d856f8213c9c43d4e is 38, key is row-t20/f:q1/1732628198342/Put/seqid=0 2024-11-26T13:36:48,616 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [256175a329f8:38807] Specified favored nodes: [256175a329f8:38807] 2024-11-26T13:36:48,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741897_1073 (size=5274) 2024-11-26T13:36:49,046 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c3ac2c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:49,046 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:49,047 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:49,048 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:49,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:49,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:49,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ced0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:49,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:49,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:49,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:49,049 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:49,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f4eb15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:49,050 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:49,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:49,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:49,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:49,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732628168449', row='row-test-1732628168449-1-0', locateType=CURRENT is [region=test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., hostname=256175a329f8,38807,1732628159830, seqNum=107] 2024-11-26T13:36:49,055 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-26T13:36:49,055 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:36:49,055 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:49,055 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:49,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:49,055 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732628168449-1-0] 2024-11-26T13:36:49,055 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 256175a329f8 2024-11-26T13:36:49,066 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344b693c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:49,066 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:49,066 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:49,067 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:49,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:49,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:49,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30032cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:49,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:49,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:49,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:49,069 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:49,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13167b6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:49,070 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:49,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:49,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:49,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:49,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732628168449', row='row-test-1732628168449-3-0', locateType=CURRENT is [region=test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., hostname=256175a329f8,38807,1732628159830, seqNum=107] 2024-11-26T13:36:49,074 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-26T13:36:49,074 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:36:49,074 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:49,074 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:49,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:49,074 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732628168449-3-0] 2024-11-26T13:36:49,074 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 256175a329f8 2024-11-26T13:36:49,083 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/backupUT/.tmp/backup_1732628199687/_temporary/0/_temporary/attempt_local239838184_0003_r_000001_0/default/test-1732628168449/f/15bf218cbd5640599de08bd1f5159d14 is 59, key is row-test-1732628168449-1-10/f:q1/1732628198010/Put/seqid=0 2024-11-26T13:36:49,085 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [256175a329f8:38807] Specified favored nodes: [256175a329f8:38807] 2024-11-26T13:36:49,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741898_1074 (size=10769) 2024-11-26T13:36:49,489 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/backupUT/.tmp/backup_1732628199687/_temporary/0/_temporary/attempt_local239838184_0003_r_000001_0/default/test-1732628168449/mob/ae235f98f60c4d37b312c4e327e328d3 is 63, key is row-test-1732628168449-3-100/mob:q1/1732628198309/Put/seqid=0 2024-11-26T13:36:49,491 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [256175a329f8:38807] Specified favored nodes: [256175a329f8:38807] 2024-11-26T13:36:49,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741899_1075 (size=11810) 2024-11-26T13:36:50,371 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:36063/backupUT 2024-11-26T13:36:50,371 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732628199687 2024-11-26T13:36:50,371 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-26T13:36:50,391 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:36063/backupUT/.tmp/backup_1732628199687, hdfs://localhost:36063/backupUT] 2024-11-26T13:36:50,467 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:36:50,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741900_1076 (size=5274) 2024-11-26T13:36:51,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741901_1077 (size=11810) 2024-11-26T13:36:51,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741902_1078 (size=10769) 2024-11-26T13:36:52,082 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-26T13:36:52,085 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732628199687 2024-11-26T13:36:52,085 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27853 bytes copied." 2024-11-26T13:36:52,086 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local220592415_0004 completed: true true 2024-11-26T13:36:52,087 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3825502 FILE: Number of bytes written=6187731 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231464 HDFS: Number of bytes written=2433179 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=250 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=339 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27853 Bytes Copied=27853 Bytes Expected=27853 Files Copied=4 DIR_COPY=8 2024-11-26T13:36:52,087 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:36063/backupUT/.tmp/backup_1732628199687 to hdfs://localhost:36063/backupUT finished. 2024-11-26T13:36:52,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:ns2:test-17326281684491:', locateType=CURRENT is [region=backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161., hostname=256175a329f8,38807,1732628159830, seqNum=2] 2024-11-26T13:36:52,102 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732628199687 2024-11-26T13:36:52,105 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732628178606} 2024-11-26T13:36:52,105 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-26T13:36:52,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741903_1079 (size=384) 2024-11-26T13:36:52,511 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36063/backupUT/backup_1732628199687/.backup.manifest 2024-11-26T13:36:52,512 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732628199687 finished: type=INCREMENTAL,tablelist=ns2:test-17326281684491;test-1732628168449,targetRootDir=hdfs://localhost:36063/backupUT,startts=1732628203953,completets=1732628212102,bytescopied=0 2024-11-26T13:36:52,512 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-26T13:36:52,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-26T13:36:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-26T13:36:52,520 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-26T13:36:52,522 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-26T13:36:52,526 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732628199687 completed. 2024-11-26T13:36:52,526 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-26T13:36:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:52,543 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:36063/backupUT/backup_1732628199687/.backup.manifest 2024-11-26T13:36:52,545 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732628199687/.backup.manifest 2024-11-26T13:36:52,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732628168449 from 'test-1732628168449', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732628168449', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:36:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732628168449 2024-11-26T13:36:52,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741904_1080 (size=908) 2024-11-26T13:36:52,962 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/.tabledesc/.tableinfo.0000000003.950 2024-11-26T13:36:52,962 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/.tabledesc/.tableinfo.0000000004.908 2024-11-26T13:36:52,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732628168449}] 2024-11-26T13:36:52,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE}] 2024-11-26T13:36:52,969 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE 2024-11-26T13:36:52,970 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=CLOSING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:52,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE because future has completed 2024-11-26T13:36:52,972 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T13:36:52,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:53,125 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,125 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T13:36:53,125 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 1d27e55c185bde71afb21ebc132e1a4f, disabling compactions & flushes 2024-11-26T13:36:53,125 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:53,125 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:53,125 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. after waiting 0 ms 2024-11-26T13:36:53,125 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:53,131 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-26T13:36:53,132 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:36:53,132 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:53,132 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 1d27e55c185bde71afb21ebc132e1a4f: Waiting for close lock at 1732628213125Running coprocessor pre-close hooks at 1732628213125Disabling compacts and flushes for region at 1732628213125Disabling writes for close at 1732628213125Writing region close event to WAL at 1732628213127 (+2 ms)Running coprocessor post-close hooks at 1732628213132 (+5 ms)Closed at 1732628213132 2024-11-26T13:36:53,132 WARN [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 1d27e55c185bde71afb21ebc132e1a4f to self. 2024-11-26T13:36:53,134 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,135 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=CLOSED 2024-11-26T13:36:53,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:53,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-26T13:36:53,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 in 165 msec 2024-11-26T13:36:53,139 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE; state=CLOSED, location=256175a329f8,38807,1732628159830; forceNewPlan=false, retain=true 2024-11-26T13:36:53,290 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:53,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE because future has completed 2024-11-26T13:36:53,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:36:53,449 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:53,449 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d27e55c185bde71afb21ebc132e1a4f, NAME => 'test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:36:53,450 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:36:53,450 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732628168449 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,450 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:36:53,451 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,451 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,453 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,454 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d27e55c185bde71afb21ebc132e1a4f columnFamilyName f 2024-11-26T13:36:53,454 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:53,462 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/497ea1f8599b476c8770fb402b3b1cd8 2024-11-26T13:36:53,467 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 2024-11-26T13:36:53,467 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(327): Store=1d27e55c185bde71afb21ebc132e1a4f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:53,467 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,469 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d27e55c185bde71afb21ebc132e1a4f columnFamilyName f2 2024-11-26T13:36:53,469 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:53,469 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(327): Store=1d27e55c185bde71afb21ebc132e1a4f/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:53,469 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,470 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d27e55c185bde71afb21ebc132e1a4f columnFamilyName mob 2024-11-26T13:36:53,471 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:36:53,477 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/21d654267d294ec594bc4a9cd1f8f382 2024-11-26T13:36:53,482 DEBUG [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 2024-11-26T13:36:53,482 INFO [StoreOpener-1d27e55c185bde71afb21ebc132e1a4f-1 {}] regionserver.HStore(327): Store=1d27e55c185bde71afb21ebc132e1a4f/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:36:53,482 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,483 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,484 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,485 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,486 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,486 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732628168449 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T13:36:53,488 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,489 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 1d27e55c185bde71afb21ebc132e1a4f; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10260926240, jitterRate=-0.04437677562236786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T13:36:53,489 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:36:53,491 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 1d27e55c185bde71afb21ebc132e1a4f: Running coprocessor pre-open hook at 1732628213451Writing region info on filesystem at 1732628213451Initializing all the Stores at 1732628213452 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628213452Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628213452Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628213452Cleaning up temporary data from old regions at 1732628213486 (+34 ms)Running coprocessor post-open hooks at 1732628213489 (+3 ms)Region opened successfully at 1732628213491 (+2 ms) 2024-11-26T13:36:53,492 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., pid=46, masterSystemTime=1732628213445 2024-11-26T13:36:53,494 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:53,494 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:36:53,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=1d27e55c185bde71afb21ebc132e1a4f, regionState=OPEN, openSeqNum=338, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:36:53,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:36:53,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-26T13:36:53,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f, server=256175a329f8,38807,1732628159830 in 205 msec 2024-11-26T13:36:53,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-26T13:36:53,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732628168449, region=1d27e55c185bde71afb21ebc132e1a4f, REOPEN/MOVE in 531 msec 2024-11-26T13:36:53,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-26T13:36:53,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732628168449 in 537 msec 2024-11-26T13:36:53,506 DEBUG [PEWorker-5 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1732628168449 2024-11-26T13:36:53,513 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f3/d47db011f3204697a9be35330503704b 2024-11-26T13:36:53,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732628168449 in 967 msec 2024-11-26T13:36:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T13:36:53,517 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732628168449 completed 2024-11-26T13:36:53,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,552 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-26T13:36:53,557 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732628213518 2024-11-26T13:36:53,559 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1732628199687} 2024-11-26T13:36:53,559 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732628178606} 2024-11-26T13:36:53,559 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-26T13:36:53,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,599 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1732628168449 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1732628168449 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T13:36:53,603 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732628213518,startts=0,failedts=1732628213600,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1732628168449 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-26T13:36:53,603 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-26T13:36:53,605 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-26T13:36:53,605 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-26T13:36:53,606 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-26T13:36:53,606 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-26T13:36:53,608 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-26T13:36:53,610 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732628213518 failed. 2024-11-26T13:36:53,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:53,626 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1732628168449, ns2:test-17326281684491, ns3:test-17326281684492, ns4:test-17326281684493] 2024-11-26T13:36:53,626 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-26T13:36:53,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:36:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628213630 (current time:1732628213630). 2024-11-26T13:36:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:36:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-26T13:36:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69224e95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:53,631 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:53,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:53,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:53,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eac2054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:53,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:53,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:53,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:53,633 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:53,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760ebcd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:53,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:53,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:53,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:53,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:53,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:36:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:53,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:36:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:36:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-26T13:36:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-26T13:36:53,639 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:36:53,640 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:36:53,643 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:36:53,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741905_1081 (size=52) 2024-11-26T13:36:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-26T13:36:53,926 DEBUG [master/256175a329f8:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732628168449 2024-11-26T13:36:53,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741906_1082 (size=950) 2024-11-26T13:36:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-26T13:36:54,049 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:36:54,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605}] 2024-11-26T13:36:54,051 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:54,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2024-11-26T13:36:54,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:54,204 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing be93584de20547dc73771fcf1a1b4605 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-26T13:36:54,208 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/1e45e166134e4296afca96b815dd2b5c is 158, key is trslm:hdfs://localhost:36063/backupUT\x00ns2:test-17326281684491/meta:log-roll-map/1732628212092/Put/seqid=0 2024-11-26T13:36:54,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741907_1083 (size=5745) 2024-11-26T13:36:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-26T13:36:54,337 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/.tabledesc/.tableinfo.0000000004.908 2024-11-26T13:36:54,337 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/.tabledesc/.tableinfo.0000000005.950 2024-11-26T13:36:54,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732628168449 in 413 msec 2024-11-26T13:36:54,613 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/1e45e166134e4296afca96b815dd2b5c 2024-11-26T13:36:54,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/b985f9302cfc4d61973fd5a8b948ccb0 is 1245, key is session:backup_1732628213518/session:context/1732628213607/Put/seqid=0 2024-11-26T13:36:54,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741908_1084 (size=6682) 2024-11-26T13:36:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-26T13:36:55,028 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/b985f9302cfc4d61973fd5a8b948ccb0 2024-11-26T13:36:55,035 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/1e45e166134e4296afca96b815dd2b5c as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c 2024-11-26T13:36:55,041 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c, entries=4, sequenceid=31, filesize=5.6 K 2024-11-26T13:36:55,042 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/b985f9302cfc4d61973fd5a8b948ccb0 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0 2024-11-26T13:36:55,047 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0, entries=3, sequenceid=31, filesize=6.5 K 2024-11-26T13:36:55,048 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for be93584de20547dc73771fcf1a1b4605 in 845ms, sequenceid=31, compaction requested=true 2024-11-26T13:36:55,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for be93584de20547dc73771fcf1a1b4605: 2024-11-26T13:36:55,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. for snapshot_backup_system completed. 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.' region-info for snapshot=snapshot_backup_system 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c] hfiles 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745 for snapshot=snapshot_backup_system 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c for snapshot=snapshot_backup_system 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0] hfiles 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 for snapshot=snapshot_backup_system 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf for snapshot=snapshot_backup_system 2024-11-26T13:36:55,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0 for snapshot=snapshot_backup_system 2024-11-26T13:36:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741909_1085 (size=251) 2024-11-26T13:36:55,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:36:55,055 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-26T13:36:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-26T13:36:55,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:55,056 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 2024-11-26T13:36:55,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=47 2024-11-26T13:36:55,058 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:36:55,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 in 1.0060 sec 2024-11-26T13:36:55,059 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:36:55,059 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:36:55,060 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-26T13:36:55,060 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-26T13:36:55,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741910_1086 (size=888) 2024-11-26T13:36:55,471 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:36:55,479 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:36:55,480 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_backup_system 2024-11-26T13:36:55,481 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:36:55,481 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-26T13:36:55,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8430 sec 2024-11-26T13:36:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-26T13:36:55,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-26T13:36:55,784 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732628213611 started at 1732628215784. 2024-11-26T13:36:55,786 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732628213611 has been started. 2024-11-26T13:36:55,790 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-26T13:36:55,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-26T13:36:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-26T13:36:55,791 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-26T13:36:55,791 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-26T13:36:55,791 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-26T13:36:55,791 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-26T13:36:55,792 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,792 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:55,794 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-26T13:36:55,794 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-26T13:36:55,794 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:36:55,795 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,795 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-26T13:36:55,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:55,795 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-26T13:36:55,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:55,796 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-26T13:36:55,796 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-26T13:36:55,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-26T13:36:55,797 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-26T13:36:55,797 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-26T13:36:55,797 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-26T13:36:55,797 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-26T13:36:55,797 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '256175a329f8,38807,1732628159830' joining acquired barrier for procedure (rolllog) in zk 2024-11-26T13:36:55,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,799 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:55,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:55,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:55,799 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-26T13:36:55,799 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-26T13:36:55,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:55,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:55,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:55,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:55,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:55,801 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '256175a329f8,38807,1732628159830' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-26T13:36:55,801 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@593716e7[Count = 0] remaining members to acquire global barrier 2024-11-26T13:36:55,801 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-26T13:36:55,802 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:55,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:55,803 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:55,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:55,803 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-26T13:36:55,803 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,803 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-26T13:36:55,803 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-26T13:36:55,803 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 256175a329f8,38807,1732628159830 2024-11-26T13:36:55,804 INFO [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732628204020 highest: 1732628204020 on 256175a329f8,38807,1732628159830 2024-11-26T13:36:55,804 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.meta:.meta(num 1732628203981) roll requested 2024-11-26T13:36:55,818 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628215804.meta, exclude list is [], retry=0 2024-11-26T13:36:55,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:55,822 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628203981.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628215804.meta 2024-11-26T13:36:55,822 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:55,822 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628203981.meta is not closed yet, will try archiving it next time 2024-11-26T13:36:55,822 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1:(num 1732628204001) roll requested 2024-11-26T13:36:55,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741889_1065 (size=1903) 2024-11-26T13:36:55,824 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628203981.meta not finished, retry = 0 2024-11-26T13:36:55,836 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628215822, exclude list is [], retry=0 2024-11-26T13:36:55,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:55,840 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628204001 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628215822 2024-11-26T13:36:55,840 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:55,840 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628204001 is not closed yet, will try archiving it next time 2024-11-26T13:36:55,840 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0:(num 1732628204020) roll requested 2024-11-26T13:36:55,841 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628184383 2024-11-26T13:36:55,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741890_1066 (size=4927) 2024-11-26T13:36:55,842 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628204001 not finished, retry = 0 2024-11-26T13:36:55,855 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841, exclude list is [], retry=0 2024-11-26T13:36:55,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:36:55,859 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628204020 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 2024-11-26T13:36:55,859 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:36:55,859 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628204020 is not closed yet, will try archiving it next time 2024-11-26T13:36:55,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741891_1067 (size=93) 2024-11-26T13:36:55,860 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628204020 not finished, retry = 0 2024-11-26T13:36:55,864 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-26T13:36:55,864 INFO [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732628215841 on 256175a329f8,38807,1732628159830 2024-11-26T13:36:55,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:36:55,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:36:55,880 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-26T13:36:55,881 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '256175a329f8,38807,1732628159830' in zk 2024-11-26T13:36:55,883 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-26T13:36:55,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,883 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-26T13:36:55,883 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-26T13:36:55,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:55,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:55,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:55,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:55,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:55,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:55,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:55,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:55,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:55,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '256175a329f8,38807,1732628159830': 2024-11-26T13:36:55,886 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '256175a329f8,38807,1732628159830' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-26T13:36:55,886 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-26T13:36:55,887 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-26T13:36:55,887 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-26T13:36:55,887 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-26T13:36:55,887 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-26T13:36:55,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-26T13:36:55,889 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,889 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,889 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:36:55,889 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,889 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:36:55,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:36:55,890 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-26T13:36:55,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:36:55,890 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:55,890 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,891 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:36:55,891 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:55,891 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:55,892 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:36:55,892 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:36:55,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:36:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-26T13:36:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-26T13:36:55,895 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-26T13:36:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:36:55,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:36:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:36:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:36:55,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-26T13:36:55,896 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-26T13:36:55,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-26T13:36:55,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-26T13:36:55,896 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-26T13:36:55,896 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:36:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-26T13:36:55,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-26T13:36:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628215906 (current time:1732628215906). 2024-11-26T13:36:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:36:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732628215906_backup_system_bulk VERSION not specified, setting to 2 2024-11-26T13:36:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6978a0bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:55,908 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:55,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:55,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:55,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca3eb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:55,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:55,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:55,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:55,910 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:55,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fb815e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:55,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:55,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:55,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:55,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:36:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:55,914 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:36:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-26T13:36:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-26T13:36:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T13:36:55,917 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:36:55,918 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:36:55,920 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:36:55,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741914_1090 (size=76) 2024-11-26T13:36:55,961 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628204020 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628204020 2024-11-26T13:36:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T13:36:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T13:36:56,327 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:36:56,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 739931d97a34c012b15574f47d4b4161}] 2024-11-26T13:36:56,328 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:56,442 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d333b64a51b935fe3679e64e25b2142a, had cached 0 bytes from a total of 8413 2024-11-26T13:36:56,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-26T13:36:56,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:56,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 739931d97a34c012b15574f47d4b4161: 2024-11-26T13:36:56,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. for snapshot_1732628215906_backup_system_bulk completed. 2024-11-26T13:36:56,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161.' region-info for snapshot=snapshot_1732628215906_backup_system_bulk 2024-11-26T13:36:56,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:36:56,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-26T13:36:56,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-26T13:36:56,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741915_1091 (size=61) 2024-11-26T13:36:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T13:36:56,888 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:36:56,888 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-26T13:36:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-26T13:36:56,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732628215906_backup_system_bulk on region 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:56,889 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 739931d97a34c012b15574f47d4b4161 2024-11-26T13:36:56,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-26T13:36:56,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 739931d97a34c012b15574f47d4b4161 in 563 msec 2024-11-26T13:36:56,892 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:36:56,892 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:36:56,893 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T13:36:56,893 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:36:56,893 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732628215906_backup_system_bulk 2024-11-26T13:36:56,894 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628215906_backup_system_bulk 2024-11-26T13:36:56,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741916_1092 (size=702) 2024-11-26T13:36:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T13:36:57,305 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:36:57,310 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:36:57,311 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628215906_backup_system_bulk to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628215906_backup_system_bulk 2024-11-26T13:36:57,312 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:36:57,312 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-26T13:36:57,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732628215906_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.3970 sec 2024-11-26T13:36:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T13:36:58,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-26T13:36:58,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } 2024-11-26T13:36:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628218064 (current time:1732628218064). 2024-11-26T13:36:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:36:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732628218064_ns2_test-17326281684491 VERSION not specified, setting to 2 2024-11-26T13:36:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362139bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:36:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:36:58,066 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:36:58,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:36:58,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:36:58,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7649d9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:58,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:36:58,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:36:58,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:58,068 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:36:58,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57a80899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:36:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:36:58,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:36:58,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:36:58,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:36:58,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:36:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:36:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:36:58,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:36:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:36:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } 2024-11-26T13:36:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-26T13:36:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T13:36:58,075 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:36:58,076 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:36:58,078 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:36:58,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741917_1093 (size=86) 2024-11-26T13:36:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T13:36:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T13:36:58,485 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:36:58,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d333b64a51b935fe3679e64e25b2142a}] 2024-11-26T13:36:58,487 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:58,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-26T13:36:58,639 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:58,639 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing d333b64a51b935fe3679e64e25b2142a 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-26T13:36:58,643 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/.tmp/f/72c5c897b12e43dc83b2c41b31d32ea0 is 38, key is row-t20/f:q1/1732628198342/Put/seqid=0 2024-11-26T13:36:58,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741918_1094 (size=5134) 2024-11-26T13:36:58,661 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-26T13:36:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T13:36:59,049 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/.tmp/f/72c5c897b12e43dc83b2c41b31d32ea0 2024-11-26T13:36:59,056 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/.tmp/f/72c5c897b12e43dc83b2c41b31d32ea0 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/72c5c897b12e43dc83b2c41b31d32ea0 2024-11-26T13:36:59,062 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/72c5c897b12e43dc83b2c41b31d32ea0, entries=5, sequenceid=111, filesize=5.0 K 2024-11-26T13:36:59,063 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for d333b64a51b935fe3679e64e25b2142a in 424ms, sequenceid=111, compaction requested=false 2024-11-26T13:36:59,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for d333b64a51b935fe3679e64e25b2142a: 2024-11-26T13:36:59,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. for snapshot_1732628218064_ns2_test-17326281684491 completed. 2024-11-26T13:36:59,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a.' region-info for snapshot=snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:36:59,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:36:59,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/72c5c897b12e43dc83b2c41b31d32ea0] hfiles 2024-11-26T13:36:59,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550 for snapshot=snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:36:59,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/72c5c897b12e43dc83b2c41b31d32ea0 for snapshot=snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:36:59,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741919_1095 (size=130) 2024-11-26T13:36:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T13:36:59,469 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:36:59,469 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-26T13:36:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-26T13:36:59,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732628218064_ns2_test-17326281684491 on region d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:59,470 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:36:59,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-26T13:36:59,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d333b64a51b935fe3679e64e25b2142a in 986 msec 2024-11-26T13:36:59,473 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:36:59,474 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:36:59,474 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:36:59,474 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:36:59,475 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:36:59,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741920_1096 (size=496) 2024-11-26T13:36:59,885 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:36:59,892 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:36:59,893 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628218064_ns2_test-17326281684491 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:36:59,894 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:36:59,894 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-26T13:36:59,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732628218064_ns2_test-17326281684491 table=ns2:test-17326281684491 type=FLUSH ttl=0 } in 1.8210 sec 2024-11-26T13:37:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T13:37:00,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17326281684491 completed 2024-11-26T13:37:00,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } 2024-11-26T13:37:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628220224 (current time:1732628220224). 2024-11-26T13:37:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:37:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732628220223_ns4_test-17326281684493 VERSION not specified, setting to 2 2024-11-26T13:37:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ad1ee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:37:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:37:00,227 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:37:00,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:37:00,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:37:00,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6960d524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:00,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:37:00,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:37:00,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:00,228 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:37:00,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e6db21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:37:00,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:37:00,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:37:00,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:37:00,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:37:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:37:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:00,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:37:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } 2024-11-26T13:37:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-26T13:37:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T13:37:00,234 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:37:00,235 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:37:00,237 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:37:00,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741921_1097 (size=86) 2024-11-26T13:37:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T13:37:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T13:37:00,644 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:37:00,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 14cd7ab999b32e4f744910187c1c85c5}] 2024-11-26T13:37:00,645 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:37:00,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-26T13:37:00,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:37:00,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 14cd7ab999b32e4f744910187c1c85c5: 2024-11-26T13:37:00,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. for snapshot_1732628220223_ns4_test-17326281684493 completed. 2024-11-26T13:37:00,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5.' region-info for snapshot=snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:00,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:37:00,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-26T13:37:00,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741922_1098 (size=52) 2024-11-26T13:37:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T13:37:01,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:37:01,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-26T13:37:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-26T13:37:01,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732628220223_ns4_test-17326281684493 on region 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:37:01,204 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:37:01,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-26T13:37:01,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 14cd7ab999b32e4f744910187c1c85c5 in 561 msec 2024-11-26T13:37:01,207 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:37:01,208 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:37:01,208 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:37:01,208 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:01,209 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:01,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741923_1099 (size=443) 2024-11-26T13:37:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T13:37:01,620 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:37:01,626 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:37:01,626 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628220223_ns4_test-17326281684493 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:01,628 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:37:01,628 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-26T13:37:01,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732628220223_ns4_test-17326281684493 table=ns4:test-17326281684493 type=FLUSH ttl=0 } in 1.3950 sec 2024-11-26T13:37:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T13:37:02,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17326281684493 completed 2024-11-26T13:37:02,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } 2024-11-26T13:37:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628222384 (current time:1732628222384). 2024-11-26T13:37:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:37:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732628222384_ns3_test-17326281684492 VERSION not specified, setting to 2 2024-11-26T13:37:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b496af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:37:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:37:02,386 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:37:02,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:37:02,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:37:02,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9088b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:02,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:37:02,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:37:02,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:02,388 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:37:02,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b27057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:37:02,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:37:02,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:37:02,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:37:02,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:37:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:37:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:02,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:37:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } 2024-11-26T13:37:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-26T13:37:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T13:37:02,395 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:37:02,395 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:37:02,398 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:37:02,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741924_1100 (size=86) 2024-11-26T13:37:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T13:37:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T13:37:02,804 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:37:02,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a62bf88ee852e800513fdadb466dea7e}] 2024-11-26T13:37:02,806 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:37:02,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-26T13:37:02,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:37:02,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for a62bf88ee852e800513fdadb466dea7e: 2024-11-26T13:37:02,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. for snapshot_1732628222384_ns3_test-17326281684492 completed. 2024-11-26T13:37:02,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e.' region-info for snapshot=snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:02,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:37:02,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-26T13:37:02,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741925_1101 (size=52) 2024-11-26T13:37:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T13:37:03,364 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:37:03,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-26T13:37:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-26T13:37:03,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732628222384_ns3_test-17326281684492 on region a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:37:03,365 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:37:03,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-26T13:37:03,368 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:37:03,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a62bf88ee852e800513fdadb466dea7e in 561 msec 2024-11-26T13:37:03,369 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:37:03,370 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:37:03,370 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:03,370 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:03,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741926_1102 (size=443) 2024-11-26T13:37:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T13:37:03,781 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:37:03,787 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:37:03,788 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628222384_ns3_test-17326281684492 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:03,789 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:37:03,789 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-26T13:37:03,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732628222384_ns3_test-17326281684492 table=ns3:test-17326281684492 type=FLUSH ttl=0 } in 1.3960 sec 2024-11-26T13:37:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T13:37:04,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17326281684492 completed 2024-11-26T13:37:04,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } 2024-11-26T13:37:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628224554 (current time:1732628224554). 2024-11-26T13:37:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:37:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732628224554_default_test-1732628168449 VERSION not specified, setting to 2 2024-11-26T13:37:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7119d048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:37:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:37:04,556 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:37:04,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:37:04,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:37:04,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc0bcce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:04,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:37:04,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:37:04,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:04,558 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:37:04,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32084c8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:37:04,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:37:04,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:37:04,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:37:04,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:37:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:37:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:04,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:37:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } 2024-11-26T13:37:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-26T13:37:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T13:37:04,565 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:37:04,566 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:37:04,568 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:37:04,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741927_1103 (size=84) 2024-11-26T13:37:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T13:37:04,775 INFO [master/256175a329f8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T13:37:04,775 INFO [master/256175a329f8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T13:37:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T13:37:04,924 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region be93584de20547dc73771fcf1a1b4605, had cached 0 bytes from a total of 28925 2024-11-26T13:37:04,975 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:37:04,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f}] 2024-11-26T13:37:04,976 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:37:05,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-26T13:37:05,128 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:37:05,128 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 1d27e55c185bde71afb21ebc132e1a4f: 2024-11-26T13:37:05,128 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. for snapshot_1732628224554_default_test-1732628168449 completed. 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.' region-info for snapshot=snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/497ea1f8599b476c8770fb402b3b1cd8] hfiles 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 for snapshot=snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/497ea1f8599b476c8770fb402b3b1cd8 for snapshot=snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/21d654267d294ec594bc4a9cd1f8f382] hfiles 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 for snapshot=snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:05,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/21d654267d294ec594bc4a9cd1f8f382 for snapshot=snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:05,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741928_1104 (size=224) 2024-11-26T13:37:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T13:37:05,535 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:37:05,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-26T13:37:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-26T13:37:05,536 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732628224554_default_test-1732628168449 on region 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:37:05,537 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:37:05,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T13:37:05,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1d27e55c185bde71afb21ebc132e1a4f in 563 msec 2024-11-26T13:37:05,539 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:37:05,540 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:37:05,541 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-26T13:37:05,541 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-26T13:37:05,542 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:05,542 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-26T13:37:05,542 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:05,542 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-26T13:37:05,543 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:05,543 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/mobdir/data/default/test-1732628168449/d2b2517e3714bd246bf40fd99916840e/mob/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f] hfiles 2024-11-26T13:37:05,544 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/mobdir/data/default/test-1732628168449/d2b2517e3714bd246bf40fd99916840e/mob/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:37:05,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741929_1105 (size=168) 2024-11-26T13:37:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T13:37:05,951 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:37:05,951 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:05,952 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:05,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741930_1106 (size=1347) 2024-11-26T13:37:06,362 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:37:06,370 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:37:06,371 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_1732628224554_default_test-1732628168449 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:06,372 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:37:06,372 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-26T13:37:06,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732628224554_default_test-1732628168449 table=test-1732628168449 type=FLUSH ttl=0 } in 1.8090 sec 2024-11-26T13:37:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T13:37:06,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732628168449 completed 2024-11-26T13:37:06,723 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732628213611 2024-11-26T13:37:06,724 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-26T13:37:06,724 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-26T13:37:06,724 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732628213611_backup:system_bulk 2024-11-26T13:37:06,724 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732628215906_backup_system_bulk to hdfs://localhost:36063/backupUT/backup_1732628213611/backup/system_bulk/ 2024-11-26T13:37:06,724 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-26T13:37:06,757 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36063, inputRoot=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:37:06,757 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1216694942_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36063/backupUT/backup_1732628213611/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36063/backupUT/backup_1732628213611/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732628215906_backup_system_bulk 2024-11-26T13:37:06,759 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-26T13:37:06,763 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628215906_backup_system_bulk to hdfs://localhost:36063/backupUT/backup_1732628213611/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732628215906_backup_system_bulk 2024-11-26T13:37:06,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741931_1107 (size=76) 2024-11-26T13:37:06,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741932_1108 (size=702) 2024-11-26T13:37:07,180 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:07,180 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:07,180 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,170 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-863932887770820152.jar 2024-11-26T13:37:08,171 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,171 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,171 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,172 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,172 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,172 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,172 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,173 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,173 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:08,173 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:37:08,173 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:37:08,174 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:37:08,174 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:37:08,174 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:37:08,174 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:37:08,174 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:37:08,175 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:37:08,175 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:37:08,175 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:37:08,175 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:37:08,176 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:08,176 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:08,176 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:08,177 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:08,177 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:08,177 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:08,177 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:08,219 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:37:08,247 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732628215906_backup_system_bulk' hfile list 2024-11-26T13:37:08,342 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1554460400_0005_a4529f20-5305-4fe0-bb0e-c74cfbc159ff/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-26T13:37:08,342 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1554460400_0005_a4529f20-5305-4fe0-bb0e-c74cfbc159ff/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-26T13:37:09,382 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-26T13:37:09,383 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-26T13:37:09,388 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732628215906_backup_system_bulk 2024-11-26T13:37:09,388 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732628215906_backup_system_bulk finished. 2024-11-26T13:37:09,388 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732628213611_ns2:test-17326281684491 2024-11-26T13:37:09,388 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732628218064_ns2_test-17326281684491 to hdfs://localhost:36063/backupUT/backup_1732628213611/ns2/test-17326281684491/ 2024-11-26T13:37:09,388 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-26T13:37:09,418 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36063, inputRoot=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:37:09,418 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1216694942_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36063/backupUT/backup_1732628213611/ns2/test-17326281684491, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36063/backupUT/backup_1732628213611/ns2/test-17326281684491/.hbase-snapshot/.tmp/snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:37:09,420 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-26T13:37:09,426 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628218064_ns2_test-17326281684491 to hdfs://localhost:36063/backupUT/backup_1732628213611/ns2/test-17326281684491/.hbase-snapshot/.tmp/snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:37:09,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741933_1109 (size=496) 2024-11-26T13:37:09,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741934_1110 (size=86) 2024-11-26T13:37:09,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:09,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:09,844 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,830 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-11109802166474344097.jar 2024-11-26T13:37:10,831 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,831 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,832 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,832 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,832 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,832 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,833 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,833 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,833 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:10,833 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:37:10,834 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:37:10,834 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:37:10,834 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:37:10,834 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:37:10,835 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:37:10,835 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:37:10,835 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:37:10,835 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:37:10,835 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:37:10,836 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:37:10,836 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:10,836 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:10,836 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:10,837 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:10,837 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:10,837 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:10,837 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:10,878 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:37:10,909 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732628218064_ns2_test-17326281684491' hfile list 2024-11-26T13:37:10,911 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-11-26T13:37:11,001 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local183876112_0006_f8977cc6-db8f-494b-921e-70f0216dda0c/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-26T13:37:11,001 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local183876112_0006_f8977cc6-db8f-494b-921e-70f0216dda0c/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-26T13:37:11,073 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-26T13:37:11,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741935_1111 (size=8413) 2024-11-26T13:37:11,517 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550 output=hdfs://localhost:36063/backupUT/backup_1732628213611/ns2/test-17326281684491/archive/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/fddff099324e4695856d43ec7e2b8550 2024-11-26T13:37:11,518 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-26T13:37:11,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741936_1112 (size=5134) 2024-11-26T13:37:11,952 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/72c5c897b12e43dc83b2c41b31d32ea0 output=hdfs://localhost:36063/backupUT/backup_1732628213611/ns2/test-17326281684491/archive/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/f/72c5c897b12e43dc83b2c41b31d32ea0 2024-11-26T13:37:11,952 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-11-26T13:37:12,042 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-26T13:37:12,043 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-26T13:37:12,049 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:37:12,050 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732628218064_ns2_test-17326281684491 finished. 2024-11-26T13:37:12,050 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732628213611_ns4:test-17326281684493 2024-11-26T13:37:12,050 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732628220223_ns4_test-17326281684493 to hdfs://localhost:36063/backupUT/backup_1732628213611/ns4/test-17326281684493/ 2024-11-26T13:37:12,050 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-26T13:37:12,080 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36063, inputRoot=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:37:12,080 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1216694942_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36063/backupUT/backup_1732628213611/ns4/test-17326281684493, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36063/backupUT/backup_1732628213611/ns4/test-17326281684493/.hbase-snapshot/.tmp/snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:12,082 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-26T13:37:12,086 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628220223_ns4_test-17326281684493 to hdfs://localhost:36063/backupUT/backup_1732628213611/ns4/test-17326281684493/.hbase-snapshot/.tmp/snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:12,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741937_1113 (size=86) 2024-11-26T13:37:12,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741938_1114 (size=443) 2024-11-26T13:37:12,498 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:12,499 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:12,499 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,486 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-18061025853924388556.jar 2024-11-26T13:37:13,487 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,487 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,487 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,488 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,488 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,488 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:13,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:37:13,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:37:13,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:37:13,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:37:13,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:37:13,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:37:13,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:37:13,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:37:13,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:37:13,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:37:13,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:37:13,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:13,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:13,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:13,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:13,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:13,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:13,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:13,534 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:37:13,561 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732628220223_ns4_test-17326281684493' hfile list 2024-11-26T13:37:13,649 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1971228069_0007_05fe4a0c-0299-49c1-a389-7dba43a11783/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-26T13:37:13,649 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1971228069_0007_05fe4a0c-0299-49c1-a389-7dba43a11783/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-26T13:37:14,688 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-26T13:37:14,689 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-26T13:37:14,693 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:14,694 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732628220223_ns4_test-17326281684493 finished. 2024-11-26T13:37:14,694 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732628213611_ns3:test-17326281684492 2024-11-26T13:37:14,694 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732628222384_ns3_test-17326281684492 to hdfs://localhost:36063/backupUT/backup_1732628213611/ns3/test-17326281684492/ 2024-11-26T13:37:14,694 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-26T13:37:14,723 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36063, inputRoot=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:37:14,723 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1216694942_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36063/backupUT/backup_1732628213611/ns3/test-17326281684492, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36063/backupUT/backup_1732628213611/ns3/test-17326281684492/.hbase-snapshot/.tmp/snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:14,725 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-26T13:37:14,729 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628222384_ns3_test-17326281684492 to hdfs://localhost:36063/backupUT/backup_1732628213611/ns3/test-17326281684492/.hbase-snapshot/.tmp/snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:14,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741939_1115 (size=86) 2024-11-26T13:37:14,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741940_1116 (size=443) 2024-11-26T13:37:15,141 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:15,141 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:15,141 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,131 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-17246356481319134476.jar 2024-11-26T13:37:16,131 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,131 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:16,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:37:16,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:37:16,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:37:16,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:37:16,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:37:16,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:37:16,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:37:16,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:37:16,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:37:16,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:37:16,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:37:16,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:16,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:16,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:16,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:16,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:16,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:16,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:16,178 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:37:16,205 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732628222384_ns3_test-17326281684492' hfile list 2024-11-26T13:37:16,293 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1758011304_0008_1f0eda6b-2a6f-465c-8c3d-5ad5e1630beb/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-26T13:37:16,293 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1758011304_0008_1f0eda6b-2a6f-465c-8c3d-5ad5e1630beb/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-26T13:37:17,333 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-26T13:37:17,334 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-26T13:37:17,340 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:17,340 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732628222384_ns3_test-17326281684492 finished. 2024-11-26T13:37:17,340 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732628213611_test-1732628168449 2024-11-26T13:37:17,340 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732628224554_default_test-1732628168449 to hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/ 2024-11-26T13:37:17,340 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-26T13:37:17,369 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36063, inputRoot=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9 2024-11-26T13:37:17,369 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1216694942_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/.hbase-snapshot/.tmp/snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:17,371 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-26T13:37:17,377 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_1732628224554_default_test-1732628168449 to hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/.hbase-snapshot/.tmp/snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:17,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741941_1117 (size=84) 2024-11-26T13:37:17,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741942_1118 (size=1347) 2024-11-26T13:37:17,790 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:17,791 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:17,791 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,779 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-16268006118895130747.jar 2024-11-26T13:37:18,779 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,780 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,780 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,780 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,780 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,781 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,781 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,781 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,781 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:18,782 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:37:18,782 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:37:18,782 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:37:18,782 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:37:18,783 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:37:18,783 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:37:18,783 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:37:18,783 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:37:18,783 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:37:18,784 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:37:18,784 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:37:18,784 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:18,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:18,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:18,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:18,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-26T13:37:18,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:18,786 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:18,826 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:37:18,852 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732628224554_default_test-1732628168449' hfile list 2024-11-26T13:37:18,854 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-11-26T13:37:18,942 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1769669704_0009_b6aa6f09-2b96-4732-b112-89a87659bc97/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-26T13:37:18,942 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/mapreduce.cluster.local.dir/job_local1769669704_0009_b6aa6f09-2b96-4732-b112-89a87659bc97/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-26T13:37:19,011 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-26T13:37:19,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741943_1119 (size=13760) 2024-11-26T13:37:19,453 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/21d654267d294ec594bc4a9cd1f8f382 output=hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/archive/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/21d654267d294ec594bc4a9cd1f8f382 2024-11-26T13:37:19,453 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-26T13:37:19,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741944_1120 (size=10796) 2024-11-26T13:37:19,886 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/497ea1f8599b476c8770fb402b3b1cd8 output=hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/archive/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/497ea1f8599b476c8770fb402b3b1cd8 2024-11-26T13:37:19,886 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-26T13:37:19,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741945_1121 (size=8413) 2024-11-26T13:37:20,328 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 output=hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/archive/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f/a9abd5f0b946478b83e74ccb81c3d433 2024-11-26T13:37:20,329 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-26T13:37:20,354 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/mobdir/data/default/test-1732628168449/d2b2517e3714bd246bf40fd99916840e/mob/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f to hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/archive/data/default/test-1732628168449/d2b2517e3714bd246bf40fd99916840e/mob/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f" truncated to max limit (512 characters) 2024-11-26T13:37:20,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741946_1122 (size=5591) 2024-11-26T13:37:20,763 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/mobdir/data/default/test-1732628168449/d2b2517e3714bd246bf40fd99916840e/mob/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f output=hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/archive/data/default/test-1732628168449/d2b2517e3714bd246bf40fd99916840e/mob/d41d8cd98f00b204e9800998ecf8427e20241126d077501f0f514511b2e571c02e7d41a3_1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:37:20,764 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.012M/sec 2024-11-26T13:37:20,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741947_1123 (size=5484) 2024-11-26T13:37:21,206 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 output=hdfs://localhost:36063/backupUT/backup_1732628213611/default/test-1732628168449/archive/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/mob/31d81ec77ef04d11b069a5ab4c7d18c5 2024-11-26T13:37:21,207 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-26T13:37:21,983 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-26T13:37:21,984 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-26T13:37:21,993 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:21,993 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732628224554_default_test-1732628168449 finished. 2024-11-26T13:37:21,993 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-26T13:37:21,993 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17326281684491 2024-11-26T13:37:21,993 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17326281684493 2024-11-26T13:37:21,993 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17326281684492 2024-11-26T13:37:21,993 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732628168449 2024-11-26T13:37:22,003 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732628213611 2024-11-26T13:37:22,003 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-26T13:37:22,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741948_1124 (size=209) 2024-11-26T13:37:22,409 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36063/backupUT/backup_1732628213611/.backup.manifest 2024-11-26T13:37:22,409 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732628213611 finished: type=FULL,tablelist=backup:system_bulk;ns2:test-17326281684491;ns4:test-17326281684493;ns3:test-17326281684492;test-1732628168449,targetRootDir=hdfs://localhost:36063/backupUT,startts=1732628215784,completets=1732628242003,bytescopied=0 2024-11-26T13:37:22,409 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-26T13:37:22,409 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732628215906_backup_system_bulk 2024-11-26T13:37:22,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732628215906_backup_system_bulk" type: DISABLED 2024-11-26T13:37:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732628215906_backup_system_bulk 2024-11-26T13:37:22,414 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732628215906_backup_system_bulk for backup backup_1732628213611 succeeded. 2024-11-26T13:37:22,414 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:37:22,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732628218064_ns2_test-17326281684491" type: DISABLED 2024-11-26T13:37:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732628218064_ns2_test-17326281684491 2024-11-26T13:37:22,417 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732628218064_ns2_test-17326281684491 for backup backup_1732628213611 succeeded. 2024-11-26T13:37:22,417 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:22,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732628220223_ns4_test-17326281684493" type: DISABLED 2024-11-26T13:37:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732628220223_ns4_test-17326281684493 2024-11-26T13:37:22,421 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732628220223_ns4_test-17326281684493 for backup backup_1732628213611 succeeded. 2024-11-26T13:37:22,421 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:22,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732628222384_ns3_test-17326281684492" type: DISABLED 2024-11-26T13:37:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732628222384_ns3_test-17326281684492 2024-11-26T13:37:22,424 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732628222384_ns3_test-17326281684492 for backup backup_1732628213611 succeeded. 2024-11-26T13:37:22,424 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:22,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732628224554_default_test-1732628168449" type: DISABLED 2024-11-26T13:37:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732628224554_default_test-1732628168449 2024-11-26T13:37:22,427 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732628224554_default_test-1732628168449 for backup backup_1732628213611 succeeded. 2024-11-26T13:37:22,428 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-26T13:37:22,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-26T13:37:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-26T13:37:22,434 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-26T13:37:22,436 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-26T13:37:22,441 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732628213611 completed. 2024-11-26T13:37:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:22,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:26,893 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T13:37:27,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,503 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-26T13:37:27,507 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732628247464 2024-11-26T13:37:27,509 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732628213611} 2024-11-26T13:37:27,509 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-26T13:37:27,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:27,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:37:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628247595 (current time:1732628247595). 2024-11-26T13:37:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:37:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-26T13:37:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53a3fd80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:37:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:37:27,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:37:27,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:37:27,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:37:27,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4675692b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:27,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:37:27,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:37:27,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:27,598 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:37:27,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ee3d00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:37:27,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:37:27,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:37:27,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:37:27,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:37:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:37:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:27,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:37:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:37:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-26T13:37:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T13:37:27,605 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:37:27,606 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:37:27,609 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:37:27,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741949_1125 (size=52) 2024-11-26T13:37:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T13:37:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T13:37:28,015 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-26T13:37:28,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605}] 2024-11-26T13:37:28,017 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:28,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-26T13:37:28,169 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:28,169 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing be93584de20547dc73771fcf1a1b4605 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-26T13:37:28,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/d6ded4b85c3e44deb2d04621202ff125 is 158, key is trslm:hdfs://localhost:36063/backupUT\x00ns2:test-17326281684491/meta:log-roll-map/1732628241998/Put/seqid=0 2024-11-26T13:37:28,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741950_1126 (size=6646) 2024-11-26T13:37:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T13:37:28,579 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/d6ded4b85c3e44deb2d04621202ff125 2024-11-26T13:37:28,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/84eb10e5568d48d381d0638dc24f6293 is 826, key is session:backup_1732628213611/session:context/1732628242435/Put/seqid=0 2024-11-26T13:37:28,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741951_1127 (size=5928) 2024-11-26T13:37:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T13:37:28,994 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/84eb10e5568d48d381d0638dc24f6293 2024-11-26T13:37:29,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/d6ded4b85c3e44deb2d04621202ff125 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125 2024-11-26T13:37:29,006 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125, entries=12, sequenceid=42, filesize=6.5 K 2024-11-26T13:37:29,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/84eb10e5568d48d381d0638dc24f6293 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293 2024-11-26T13:37:29,013 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293, entries=2, sequenceid=42, filesize=5.8 K 2024-11-26T13:37:29,015 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for be93584de20547dc73771fcf1a1b4605 in 845ms, sequenceid=42, compaction requested=true 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for be93584de20547dc73771fcf1a1b4605: 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. for snapshot_backup_system completed. 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.' region-info for snapshot=snapshot_backup_system 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125] hfiles 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745 for snapshot=snapshot_backup_system 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c for snapshot=snapshot_backup_system 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125 for snapshot=snapshot_backup_system 2024-11-26T13:37:29,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293] hfiles 2024-11-26T13:37:29,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 for snapshot=snapshot_backup_system 2024-11-26T13:37:29,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf for snapshot=snapshot_backup_system 2024-11-26T13:37:29,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0 for snapshot=snapshot_backup_system 2024-11-26T13:37:29,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293 for snapshot=snapshot_backup_system 2024-11-26T13:37:29,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741952_1128 (size=330) 2024-11-26T13:37:29,422 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:29,423 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-26T13:37:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-26T13:37:29,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:29,424 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:29,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T13:37:29,426 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-26T13:37:29,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure be93584de20547dc73771fcf1a1b4605 in 1.4090 sec 2024-11-26T13:37:29,427 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:37:29,427 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:37:29,427 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-26T13:37:29,428 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-26T13:37:29,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741953_1129 (size=967) 2024-11-26T13:37:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T13:37:29,838 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:37:29,847 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:37:29,847 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/snapshot_backup_system 2024-11-26T13:37:29,849 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:37:29,849 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-26T13:37:29,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2460 sec 2024-11-26T13:37:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T13:37:31,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-26T13:37:31,783 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732628247464 started at 1732628251783. 2024-11-26T13:37:31,786 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732628247464 has been started. 2024-11-26T13:37:31,790 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, test-1732628168449, ns2:test-17326281684491, ns3:test-17326281684492, ns4:test-17326281684493] 2024-11-26T13:37:31,794 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732628204020for backupID backup_1732628247464 2024-11-26T13:37:31,794 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-26T13:37:31,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-26T13:37:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-26T13:37:31,795 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-26T13:37:31,795 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-26T13:37:31,795 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-26T13:37:31,795 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-26T13:37:31,796 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,796 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-26T13:37:31,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-26T13:37:31,802 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,803 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-26T13:37:31,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:37:31,803 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,803 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-26T13:37:31,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-26T13:37:31,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-26T13:37:31,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-26T13:37:31,804 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-26T13:37:31,805 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-26T13:37:31,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-26T13:37:31,805 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-26T13:37:31,805 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-26T13:37:31,805 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-26T13:37:31,805 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-26T13:37:31,805 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '256175a329f8,38807,1732628159830' joining acquired barrier for procedure (rolllog) in zk 2024-11-26T13:37:31,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,807 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-26T13:37:31,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:37:31,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:37:31,808 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-26T13:37:31,808 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-26T13:37:31,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:37:31,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:37:31,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:37:31,809 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:37:31,809 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:37:31,810 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '256175a329f8,38807,1732628159830' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-26T13:37:31,810 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5c67adab[Count = 0] remaining members to acquire global barrier 2024-11-26T13:37:31,810 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-26T13:37:31,810 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-26T13:37:31,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:37:31,811 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-26T13:37:31,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-26T13:37:31,812 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-26T13:37:31,812 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,812 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-26T13:37:31,812 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-26T13:37:31,812 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 256175a329f8,38807,1732628159830 2024-11-26T13:37:31,812 INFO [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732628215841 highest: 1732628215841 on 256175a329f8,38807,1732628159830 2024-11-26T13:37:31,812 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.meta:.meta(num 1732628215804) roll requested 2024-11-26T13:37:31,826 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628251812.meta, exclude list is [], retry=0 2024-11-26T13:37:31,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:37:31,831 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628215804.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628251812.meta 2024-11-26T13:37:31,831 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:37:31,831 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628215804.meta is not closed yet, will try archiving it next time 2024-11-26T13:37:31,831 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1:(num 1732628215822) roll requested 2024-11-26T13:37:31,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741911_1087 (size=93) 2024-11-26T13:37:31,832 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628215804.meta not finished, retry = 0 2024-11-26T13:37:31,845 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628251831, exclude list is [], retry=0 2024-11-26T13:37:31,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:37:31,848 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628215822 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628251831 2024-11-26T13:37:31,849 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:37:31,849 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628215822 is not closed yet, will try archiving it next time 2024-11-26T13:37:31,849 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0:(num 1732628215841) roll requested 2024-11-26T13:37:31,849 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628204001 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628204001 2024-11-26T13:37:31,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741912_1088 (size=4710) 2024-11-26T13:37:31,863 DEBUG [regionserver/256175a329f8:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628251849, exclude list is [], retry=0 2024-11-26T13:37:31,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46229,DS-54da41b9-cdfd-4ebc-8ccd-e1bd41441124,DISK] 2024-11-26T13:37:31,867 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628251849 2024-11-26T13:37:31,867 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-26T13:37:31,867 DEBUG [regionserver/256175a329f8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 is not closed yet, will try archiving it next time 2024-11-26T13:37:31,867 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628184406 2024-11-26T13:37:31,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741913_1089 (size=653) 2024-11-26T13:37:31,869 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 not finished, retry = 0 2024-11-26T13:37:31,873 DEBUG [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-26T13:37:31,873 INFO [rs(256175a329f8,38807,1732628159830)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732628251849 on 256175a329f8,38807,1732628159830 2024-11-26T13:37:31,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-26T13:37:31,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:31,892 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-26T13:37:31,892 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '256175a329f8,38807,1732628159830' in zk 2024-11-26T13:37:31,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,894 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-26T13:37:31,895 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-26T13:37:31,895 DEBUG [member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-26T13:37:31,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:37:31,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:37:31,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:37:31,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:37:31,896 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:37:31,896 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:37:31,897 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:37:31,897 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:37:31,897 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:37:31,898 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '256175a329f8,38807,1732628159830': 2024-11-26T13:37:31,898 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-26T13:37:31,898 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '256175a329f8,38807,1732628159830' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-26T13:37:31,898 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-26T13:37:31,898 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-26T13:37:31,898 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-26T13:37:31,898 INFO [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-26T13:37:31,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-26T13:37:31,900 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,900 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,900 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,900 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-26T13:37:31,900 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-26T13:37:31,901 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-26T13:37:31,901 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-26T13:37:31,901 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:37:31,901 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,901 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:37:31,901 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-26T13:37:31,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:37:31,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:37:31,903 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,903 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-26T13:37:31,903 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-26T13:37:31,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------256175a329f8,38807,1732628159830 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/256175a329f8,38807,1732628159830 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-26T13:37:31,906 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-26T13:37:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-26T13:37:31,906 DEBUG [(256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-26T13:37:31,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-26T13:37:31,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-26T13:37:31,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-26T13:37:31,907 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-26T13:37:31,907 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-26T13:37:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-26T13:37:31,915 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {256175a329f8:38807=1732628204020} newestTimestamps: {256175a329f8:38807=1732628215841} 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 256175a329f8%2C38807%2C1732628159830.meta.1732628163597.meta 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 256175a329f8%2C38807%2C1732628159830.meta.1732628203981.meta 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 256175a329f8%2C38807%2C1732628159830.meta.1732628215804.meta 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 256175a329f8%2C38807%2C1732628159830.meta.1732628251812.meta 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628251849 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628215822 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628251831 2024-11-26T13:37:31,917 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.meta.1732628184362.meta 2024-11-26T13:37:31,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741957_1133 (size=640) 2024-11-26T13:37:31,933 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628215804.meta to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.meta.1732628215804.meta 2024-11-26T13:37:31,970 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 2024-11-26T13:37:32,328 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:36063/backupUT/backup_1732628247464/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:37:32,328 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-26T13:37:32,330 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-26T13:37:32,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741958_1134 (size=45) 2024-11-26T13:37:32,736 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-26T13:37:32,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741959_1135 (size=364) 2024-11-26T13:37:33,147 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17326281684491 target: hdfs://localhost:36063/backupUT/backup_1732628247464/ns2/test-17326281684491 descriptor: 'ns2:test-17326281684491', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:37:33,147 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-26T13:37:33,149 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17326281684491 2024-11-26T13:37:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741960_1136 (size=50) 2024-11-26T13:37:33,555 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17326281684491 2024-11-26T13:37:33,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741961_1137 (size=390) 2024-11-26T13:37:33,966 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17326281684493 target: hdfs://localhost:36063/backupUT/backup_1732628247464/ns4/test-17326281684493 descriptor: 'ns4:test-17326281684493', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:37:33,966 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-26T13:37:33,968 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17326281684493 2024-11-26T13:37:33,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741962_1138 (size=50) 2024-11-26T13:37:34,374 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17326281684493 2024-11-26T13:37:34,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741963_1139 (size=390) 2024-11-26T13:37:34,785 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17326281684492 target: hdfs://localhost:36063/backupUT/backup_1732628247464/ns3/test-17326281684492 descriptor: 'ns3:test-17326281684492', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:37:34,785 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-26T13:37:34,787 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17326281684492 2024-11-26T13:37:34,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741964_1140 (size=50) 2024-11-26T13:37:35,193 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17326281684492 2024-11-26T13:37:35,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741965_1141 (size=950) 2024-11-26T13:37:35,604 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732628168449 target: hdfs://localhost:36063/backupUT/backup_1732628247464/default/test-1732628168449 descriptor: 'test-1732628168449', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T13:37:35,604 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-26T13:37:35,606 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732628168449 2024-11-26T13:37:35,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741966_1142 (size=53) 2024-11-26T13:37:36,012 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732628168449 2024-11-26T13:37:36,022 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:36063/backupUT/.tmp/backup_1732628247464 from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841;hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628215822 2024-11-26T13:37:36,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3218c40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:36,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:37:36,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:37:36,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:37:36,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:37:36,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:37:36,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688efc84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:36,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:37:36,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:37:36,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:36,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:37:36,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3259f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:36,027 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T13:37:36,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 256175a329f8,40825,1732628157903 2024-11-26T13:37:36,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76a6738c 2024-11-26T13:37:36,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T13:37:36,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T13:37:36,032 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-26T13:37:36,032 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-26T13:37:36,032 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-26T13:37:36,032 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:37:36,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:37:36,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:37:36,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:37:36,036 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-26T13:37:36,036 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732628168449 2024-11-26T13:37:36,036 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732628168449,, stopping at row=test-1732628168449 ,, for max=2147483647 with caching=100 2024-11-26T13:37:36,038 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732628168449: test-1732628168449; 2024-11-26T13:37:36,038 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17326281684491 2024-11-26T13:37:36,038 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17326281684491,, stopping at row=ns2:test-17326281684491 ,, for max=2147483647 with caching=100 2024-11-26T13:37:36,039 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17326281684491: ns2:test-17326281684491; 2024-11-26T13:37:36,039 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17326281684492 2024-11-26T13:37:36,039 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17326281684492,, stopping at row=ns3:test-17326281684492 ,, for max=2147483647 with caching=100 2024-11-26T13:37:36,041 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17326281684492: ns3:test-17326281684492; 2024-11-26T13:37:36,041 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17326281684493 2024-11-26T13:37:36,041 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17326281684493,, stopping at row=ns4:test-17326281684493 ,, for max=2147483647 with caching=100 2024-11-26T13:37:36,042 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17326281684493: ns4:test-17326281684493; 2024-11-26T13:37:36,042 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-26T13:37:36,042 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/hbase-staging/partitions_e7430c4c-a4c2-4681-aa96-d1f4bf02073a 2024-11-26T13:37:36,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741967_1143 (size=324) 2024-11-26T13:37:36,450 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:36,450 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:36,450 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,441 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-7978471949665051027.jar 2024-11-26T13:37:37,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:37:37,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-26T13:37:37,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-26T13:37:37,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-26T13:37:37,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-26T13:37:37,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-26T13:37:37,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-26T13:37:37,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-26T13:37:37,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-26T13:37:37,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-26T13:37:37,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-26T13:37:37,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,448 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,448 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,448 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,448 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,449 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-26T13:37:37,449 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-26T13:37:37,449 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,default:test-1732628168449,ns2:test-17326281684491,ns3:test-17326281684492,ns4:test-17326281684493 2024-11-26T13:37:37,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T13:37:37,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:37:37,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:37,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:37,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:37,450 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-26T13:37:38,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop-9670836312924749183.jar 2024-11-26T13:37:38,451 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1d27e55c185bde71afb21ebc132e1a4f, had cached 0 bytes from a total of 38453 2024-11-26T13:37:38,455 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-26T13:37:38,481 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 for WAL files 2024-11-26T13:37:38,482 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1732628247464File hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 does not exist. java.io.FileNotFoundException: File hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T13:37:38,483 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732628247464,startts=1732628251783,failedts=1732628258483,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628215841 does not exist. 2024-11-26T13:37:38,483 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-26T13:37:38,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable backup:system 2024-11-26T13:37:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-11-26T13:37:38,501 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628258501"}]},"ts":"1732628258501"} 2024-11-26T13:37:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T13:37:38,502 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-11-26T13:37:38,502 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-11-26T13:37:38,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-11-26T13:37:38,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, UNASSIGN}] 2024-11-26T13:37:38,509 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, UNASSIGN 2024-11-26T13:37:38,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=be93584de20547dc73771fcf1a1b4605, regionState=CLOSING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:37:38,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, UNASSIGN because future has completed 2024-11-26T13:37:38,512 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T13:37:38,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:37:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T13:37:38,664 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:38,664 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T13:37:38,664 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing be93584de20547dc73771fcf1a1b4605, disabling compactions & flushes 2024-11-26T13:37:38,664 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:38,665 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:38,665 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. after waiting 0 ms 2024-11-26T13:37:38,665 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:38,665 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing be93584de20547dc73771fcf1a1b4605 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-11-26T13:37:38,670 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/024c614e57cb4820a6f79d5ef1e40478 is 103, key is rslogts:hdfs://localhost:36063/backupUT\x00256175a329f8:38807/meta:rs-log-ts/1732628251891/Put/seqid=0 2024-11-26T13:37:38,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741968_1144 (size=5338) 2024-11-26T13:37:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T13:37:39,076 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/024c614e57cb4820a6f79d5ef1e40478 2024-11-26T13:37:39,085 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/a06d708ff65c4af38d050052a97d6e99 is 303, key is session:backup_1732628247464/session:context/1732628251785/Put/seqid=0 2024-11-26T13:37:39,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741969_1145 (size=5391) 2024-11-26T13:37:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T13:37:39,489 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/a06d708ff65c4af38d050052a97d6e99 2024-11-26T13:37:39,495 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/024c614e57cb4820a6f79d5ef1e40478 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/024c614e57cb4820a6f79d5ef1e40478 2024-11-26T13:37:39,500 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/024c614e57cb4820a6f79d5ef1e40478, entries=1, sequenceid=47, filesize=5.2 K 2024-11-26T13:37:39,501 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/a06d708ff65c4af38d050052a97d6e99 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/a06d708ff65c4af38d050052a97d6e99 2024-11-26T13:37:39,507 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/a06d708ff65c4af38d050052a97d6e99, entries=1, sequenceid=47, filesize=5.3 K 2024-11-26T13:37:39,508 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for be93584de20547dc73771fcf1a1b4605 in 843ms, sequenceid=47, compaction requested=true 2024-11-26T13:37:39,512 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-11-26T13:37:39,513 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:37:39,513 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:39,513 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for be93584de20547dc73771fcf1a1b4605: Waiting for close lock at 1732628258664Running coprocessor pre-close hooks at 1732628258664Disabling compacts and flushes for region at 1732628258664Disabling writes for close at 1732628258665 (+1 ms)Obtaining lock to block concurrent updates at 1732628258665Preparing flush snapshotting stores in be93584de20547dc73771fcf1a1b4605 at 1732628258665Finished memstore snapshotting backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1732628258665Flushing stores of backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. at 1732628258666 (+1 ms)Flushing be93584de20547dc73771fcf1a1b4605/meta: creating writer at 1732628258666Flushing be93584de20547dc73771fcf1a1b4605/meta: appending metadata at 1732628258669 (+3 ms)Flushing be93584de20547dc73771fcf1a1b4605/meta: closing flushed file at 1732628258670 (+1 ms)Flushing be93584de20547dc73771fcf1a1b4605/session: creating writer at 1732628259081 (+411 ms)Flushing be93584de20547dc73771fcf1a1b4605/session: appending metadata at 1732628259084 (+3 ms)Flushing be93584de20547dc73771fcf1a1b4605/session: closing flushed file at 1732628259084Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9ed80b9: reopening flushed file at 1732628259494 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@593c5b55: reopening flushed file at 1732628259501 (+7 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for be93584de20547dc73771fcf1a1b4605 in 843ms, sequenceid=47, compaction requested=true at 1732628259508 (+7 ms)Writing region close event to WAL at 1732628259509 (+1 ms)Running coprocessor post-close hooks at 1732628259513 (+4 ms)Closed at 1732628259513 2024-11-26T13:37:39,515 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:39,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=be93584de20547dc73771fcf1a1b4605, regionState=CLOSED 2024-11-26T13:37:39,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:37:39,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T13:37:39,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830 in 1.0070 sec 2024-11-26T13:37:39,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-26T13:37:39,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, UNASSIGN in 1.0120 sec 2024-11-26T13:37:39,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T13:37:39,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0190 sec 2024-11-26T13:37:39,525 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628259525"}]},"ts":"1732628259525"} 2024-11-26T13:37:39,527 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-11-26T13:37:39,527 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-11-26T13:37:39,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0330 sec 2024-11-26T13:37:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T13:37:39,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-11-26T13:37:39,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732628259652 2024-11-26T13:37:39,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:37:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732628259653 (current time:1732628259653). 2024-11-26T13:37:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-26T13:37:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1732628259652 VERSION not specified, setting to 2 2024-11-26T13:37:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ece09e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ClusterIdFetcher(90): Going to request 256175a329f8,40825,-1 for getting cluster id 2024-11-26T13:37:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T13:37:39,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722448d4-7b7c-4464-9023-558761cfe9fd' 2024-11-26T13:37:39,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T13:37:39,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722448d4-7b7c-4464-9023-558761cfe9fd" 2024-11-26T13:37:39,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73900ec8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:39,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [256175a329f8,40825,-1] 2024-11-26T13:37:39,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T13:37:39,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:39,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T13:37:39,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732115f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T13:37:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T13:37:39,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=256175a329f8,38807,1732628159830, seqNum=-1] 2024-11-26T13:37:39,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T13:37:39,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T13:37:39,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825. 2024-11-26T13:37:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T13:37:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:39,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-26T13:37:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:37:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-26T13:37:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T13:37:39,662 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-26T13:37:39,663 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-26T13:37:39,666 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-26T13:37:39,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741970_1146 (size=81) 2024-11-26T13:37:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T13:37:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T13:37:40,072 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-11-26T13:37:40,073 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-26T13:37:40,073 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-26T13:37:40,073 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:40,075 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/024c614e57cb4820a6f79d5ef1e40478, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745] hfiles 2024-11-26T13:37:40,075 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/024c614e57cb4820a6f79d5ef1e40478 2024-11-26T13:37:40,075 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c 2024-11-26T13:37:40,075 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125 2024-11-26T13:37:40,075 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745 2024-11-26T13:37:40,075 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:40,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/a06d708ff65c4af38d050052a97d6e99, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4] hfiles 2024-11-26T13:37:40,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293 2024-11-26T13:37:40,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf 2024-11-26T13:37:40,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/a06d708ff65c4af38d050052a97d6e99 2024-11-26T13:37:40,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0 2024-11-26T13:37:40,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 2024-11-26T13:37:40,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741971_1147 (size=409) 2024-11-26T13:37:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T13:37:40,483 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => be93584de20547dc73771fcf1a1b4605, NAME => 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-11-26T13:37:40,484 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-26T13:37:40,485 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-26T13:37:40,485 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1732628259652 2024-11-26T13:37:40,486 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732628259652 2024-11-26T13:37:40,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741972_1148 (size=1046) 2024-11-26T13:37:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T13:37:40,896 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-26T13:37:40,905 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-26T13:37:40,905 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732628259652 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1732628259652 2024-11-26T13:37:40,906 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-26T13:37:40,907 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-26T13:37:40,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732628259652 table=backup:system type=FLUSH ttl=0 } in 1.2460 sec 2024-11-26T13:37:41,442 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d333b64a51b935fe3679e64e25b2142a, had cached 0 bytes from a total of 13547 2024-11-26T13:37:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T13:37:41,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-26T13:37:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732628247595 type: FLUSH version: 2 ttl: 0 ) 2024-11-26T13:37:41,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-11-26T13:37:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T13:37:41,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741973_1149 (size=635) 2024-11-26T13:37:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T13:37:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T13:37:42,250 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-11-26T13:37:42,250 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-11-26T13:37:42,252 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-26T13:37:42,256 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732628247595 type: FLUSH version: 2 ttl: 0 2024-11-26T13:37:42,257 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system 2024-11-26T13:37:42,260 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-11-26T13:37:42,260 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(229): region to restore: be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,264 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:42,267 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/024c614e57cb4820a6f79d5ef1e40478 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/024c614e57cb4820a6f79d5ef1e40478 2024-11-26T13:37:42,268 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:42,271 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/a06d708ff65c4af38d050052a97d6e99 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/a06d708ff65c4af38d050052a97d6e99 2024-11-26T13:37:42,271 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732628247595 type: FLUSH version: 2 ttl: 0 2024-11-26T13:37:42,273 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732628262272"}]},"ts":"9223372036854775807"} 2024-11-26T13:37:42,276 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T13:37:42,276 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => be93584de20547dc73771fcf1a1b4605, NAME => 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T13:37:42,276 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732628262273"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732628262273"}]},"ts":"1732628262273"} 2024-11-26T13:37:42,278 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T13:37:42,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-11-26T13:37:42,278 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => be93584de20547dc73771fcf1a1b4605, NAME => 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T13:37:42,278 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-11-26T13:37:42,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732628247595 type: FLUSH version: 2 ttl: 0 ) in 451 msec 2024-11-26T13:37:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-26T13:37:42,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732628259652 2024-11-26T13:37:42,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "hbase-failsafe-snapshot_backup_system-1732628259652" type: DISABLED 2024-11-26T13:37:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1732628259652 2024-11-26T13:37:42,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.2 enable backup:system 2024-11-26T13:37:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-11-26T13:37:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T13:37:42,479 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-11-26T13:37:42,479 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628262479"}]},"ts":"1732628262479"} 2024-11-26T13:37:42,481 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-26T13:37:42,482 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-11-26T13:37:42,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN}] 2024-11-26T13:37:42,483 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN 2024-11-26T13:37:42,483 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-11-26T13:37:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T13:37:42,634 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=be93584de20547dc73771fcf1a1b4605, regionState=OPENING, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:37:42,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN because future has completed 2024-11-26T13:37:42,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830}] 2024-11-26T13:37:42,792 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:42,792 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => be93584de20547dc73771fcf1a1b4605, NAME => 'backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.', STARTKEY => '', ENDKEY => ''} 2024-11-26T13:37:42,793 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-26T13:37:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T13:37:42,793 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,793 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T13:37:42,793 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,793 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,795 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,795 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be93584de20547dc73771fcf1a1b4605 columnFamilyName meta 2024-11-26T13:37:42,796 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:42,802 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c 2024-11-26T13:37:42,806 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125 2024-11-26T13:37:42,810 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745 2024-11-26T13:37:42,810 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.HStore(327): Store=be93584de20547dc73771fcf1a1b4605/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:37:42,810 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,811 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be93584de20547dc73771fcf1a1b4605 columnFamilyName session 2024-11-26T13:37:42,811 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T13:37:42,817 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293 2024-11-26T13:37:42,821 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf 2024-11-26T13:37:42,824 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0 2024-11-26T13:37:42,828 DEBUG [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 2024-11-26T13:37:42,828 INFO [StoreOpener-be93584de20547dc73771fcf1a1b4605-1 {}] regionserver.HStore(327): Store=be93584de20547dc73771fcf1a1b4605/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T13:37:42,829 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,829 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,831 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,831 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,831 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,832 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T13:37:42,834 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,834 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened be93584de20547dc73771fcf1a1b4605; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9569178080, jitterRate=-0.10880084335803986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T13:37:42,835 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:42,835 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for be93584de20547dc73771fcf1a1b4605: Running coprocessor pre-open hook at 1732628262793Writing region info on filesystem at 1732628262793Initializing all the Stores at 1732628262794 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628262794Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732628262794Cleaning up temporary data from old regions at 1732628262831 (+37 ms)Running coprocessor post-open hooks at 1732628262835 (+4 ms)Region opened successfully at 1732628262835 2024-11-26T13:37:42,836 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., pid=70, masterSystemTime=1732628262788 2024-11-26T13:37:42,838 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store be93584de20547dc73771fcf1a1b4605:meta, priority=-2147483648, current under compaction store size is 1 2024-11-26T13:37:42,840 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T13:37:42,840 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T13:37:42,840 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store be93584de20547dc73771fcf1a1b4605:session, priority=-2147483648, current under compaction store size is 2 2024-11-26T13:37:42,840 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T13:37:42,840 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T13:37:42,842 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T13:37:42,842 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T13:37:42,842 DEBUG [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:42,842 INFO [RS_OPEN_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:42,842 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=be93584de20547dc73771fcf1a1b4605, regionState=OPEN, openSeqNum=51, regionLocation=256175a329f8,38807,1732628159830 2024-11-26T13:37:42,843 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.HStore(1541): be93584de20547dc73771fcf1a1b4605/session is initiating minor compaction (all files) 2024-11-26T13:37:42,843 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.HStore(1541): be93584de20547dc73771fcf1a1b4605/meta is initiating minor compaction (all files) 2024-11-26T13:37:42,844 INFO [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be93584de20547dc73771fcf1a1b4605/meta in backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:42,844 INFO [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be93584de20547dc73771fcf1a1b4605/session in backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:42,844 INFO [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125] into tmpdir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp, totalSize=17.9 K 2024-11-26T13:37:42,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830 because future has completed 2024-11-26T13:37:42,844 INFO [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293] into tmpdir=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp, totalSize=22.7 K 2024-11-26T13:37:42,845 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] compactions.Compactor(225): Compacting d39591ef2cef4c0299ebddb16931dca4, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732628182116 2024-11-26T13:37:42,845 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef83da5ee1434703a7a27938ed98d745, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732628184452 2024-11-26T13:37:42,846 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e45e166134e4296afca96b815dd2b5c, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732628204060 2024-11-26T13:37:42,846 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] compactions.Compactor(225): Compacting 87d571180ca341c4a22fd473d47010cf, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732628197951 2024-11-26T13:37:42,846 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6ded4b85c3e44deb2d04621202ff125, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732628215880 2024-11-26T13:37:42,846 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] compactions.Compactor(225): Compacting b985f9302cfc4d61973fd5a8b948ccb0, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732628212521 2024-11-26T13:37:42,847 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] compactions.Compactor(225): Compacting 84eb10e5568d48d381d0638dc24f6293, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732628242435 2024-11-26T13:37:42,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-26T13:37:42,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure be93584de20547dc73771fcf1a1b4605, server=256175a329f8,38807,1732628159830 in 208 msec 2024-11-26T13:37:42,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-26T13:37:42,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=be93584de20547dc73771fcf1a1b4605, ASSIGN in 365 msec 2024-11-26T13:37:42,850 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732628262850"}]},"ts":"1732628262850"} 2024-11-26T13:37:42,852 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-26T13:37:42,852 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-11-26T13:37:42,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 380 msec 2024-11-26T13:37:42,874 INFO [RS:0;256175a329f8:38807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be93584de20547dc73771fcf1a1b4605#meta#compaction#16 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T13:37:42,874 INFO [RS:0;256175a329f8:38807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be93584de20547dc73771fcf1a1b4605#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T13:37:42,875 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/e8fb120c7dde4b2da06171453a176148 is 1245, key is session:backup_1732628213518/session:context/1732628213607/Put/seqid=0 2024-11-26T13:37:42,876 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/8f7127b1d85e40048c59ca9db679a4d3 is 158, key is trslm:hdfs://localhost:36063/backupUT\x00ns2:test-17326281684491/meta:log-roll-map/1732628241998/Put/seqid=0 2024-11-26T13:37:42,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741974_1150 (size=6749) 2024-11-26T13:37:42,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741975_1151 (size=8073) 2024-11-26T13:37:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T13:37:43,104 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-11-26T13:37:43,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-11-26T13:37:43,104 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-26T13:37:43,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-26T13:37:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40825 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-26T13:37:43,111 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-26T13:37:43,111 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-11-26T13:37:43,113 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-26T13:37:43,118 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732628247464 failed. 2024-11-26T13:37:43,118 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T13:37:43,118 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:37:43,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:43,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:43,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:43,139 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=627 (was 611) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/e8fb120c7dde4b2da06171453a176148 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(256175a329f8,38807,1732628159830)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(256175a329f8,38807,1732628159830)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/8f7127b1d85e40048c59ca9db679a4d3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: RS_CLOSE_REGION-regionserver/256175a329f8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-992671285_20 at /127.0.0.1:33490 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (256175a329f8,40825,1732628157903)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-650579929-172.17.0.2-1732628154247:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-650579929-172.17.0.2-1732628154247:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: '256175a329f8,38807,1732628159830' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;256175a329f8:38807-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(256175a329f8,38807,1732628159830)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-650579929-172.17.0.2-1732628154247:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(256175a329f8,38807,1732628159830)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-546230263_20 at /127.0.0.1:39342 [Receiving block BP-650579929-172.17.0.2-1732628154247:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-546230263_20 at /127.0.0.1:39360 [Receiving block BP-650579929-172.17.0.2-1732628154247:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/256175a329f8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-546230263_20 at /127.0.0.1:39352 [Receiving block BP-650579929-172.17.0.2-1732628154247:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=641 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=109 (was 222), ProcessCount=16 (was 16), AvailableMemoryMB=4837 (was 5331) 2024-11-26T13:37:43,140 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=627 is superior to 500 2024-11-26T13:37:43,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T13:37:43,147 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T13:37:43,147 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:37:43,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:43,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:43,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:43,147 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T13:37:43,147 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1132182242, stopped=false 2024-11-26T13:37:43,148 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=256175a329f8,40825,1732628157903 2024-11-26T13:37:43,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-26T13:37:43,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-26T13:37:43,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:37:43,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:37:43,149 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T13:37:43,150 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T13:37:43,150 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:37:43,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-26T13:37:43,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-26T13:37:43,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:43,151 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '256175a329f8,38807,1732628159830' ***** 2024-11-26T13:37:43,151 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T13:37:43,151 INFO [RS:0;256175a329f8:38807 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-26T13:37:43,155 INFO [RS:0;256175a329f8:38807 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@6e12a44d{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-26T13:37:43,157 INFO [RS:0;256175a329f8:38807 {}] server.AbstractConnector(383): Stopped ServerConnector@5804e71e{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-26T13:37:43,157 INFO [RS:0;256175a329f8:38807 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T13:37:43,157 INFO [RS:0;256175a329f8:38807 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@23a380bc{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-26T13:37:43,157 INFO [RS:0;256175a329f8:38807 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@13c18cd6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,STOPPED} 2024-11-26T13:37:43,159 INFO [RS:0;256175a329f8:38807 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T13:37:43,159 INFO [RS:0;256175a329f8:38807 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T13:37:43,159 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T13:37:43,159 INFO [RS:0;256175a329f8:38807 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-26T13:37:43,159 INFO [RS:0;256175a329f8:38807 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T13:37:43,159 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for d333b64a51b935fe3679e64e25b2142a 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 14cd7ab999b32e4f744910187c1c85c5 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 739931d97a34c012b15574f47d4b4161 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for a62bf88ee852e800513fdadb466dea7e 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(959): stopping server 256175a329f8,38807,1732628159830 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;256175a329f8:38807. 2024-11-26T13:37:43,160 DEBUG [RS:0;256175a329f8:38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:37:43,160 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d333b64a51b935fe3679e64e25b2142a, disabling compactions & flushes 2024-11-26T13:37:43,160 DEBUG [RS:0;256175a329f8:38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:43,160 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:37:43,160 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:37:43,160 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. after waiting 0 ms 2024-11-26T13:37:43,160 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:37:43,160 INFO [RS:0;256175a329f8:38807 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T13:37:43,165 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns2/test-17326281684491/d333b64a51b935fe3679e64e25b2142a/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-26T13:37:43,165 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:37:43,166 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:37:43,166 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d333b64a51b935fe3679e64e25b2142a: Waiting for close lock at 1732628263160Running coprocessor pre-close hooks at 1732628263160Disabling compacts and flushes for region at 1732628263160Disabling writes for close at 1732628263160Writing region close event to WAL at 1732628263161 (+1 ms)Running coprocessor post-close hooks at 1732628263165 (+4 ms)Closed at 1732628263166 (+1 ms) 2024-11-26T13:37:43,166 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a. 2024-11-26T13:37:43,166 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 14cd7ab999b32e4f744910187c1c85c5, disabling compactions & flushes 2024-11-26T13:37:43,166 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:37:43,166 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:37:43,166 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. after waiting 0 ms 2024-11-26T13:37:43,166 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:37:43,169 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns4/test-17326281684493/14cd7ab999b32e4f744910187c1c85c5/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-26T13:37:43,170 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:37:43,170 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:37:43,170 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 14cd7ab999b32e4f744910187c1c85c5: Waiting for close lock at 1732628263166Running coprocessor pre-close hooks at 1732628263166Disabling compacts and flushes for region at 1732628263166Disabling writes for close at 1732628263166Writing region close event to WAL at 1732628263166Running coprocessor post-close hooks at 1732628263170 (+4 ms)Closed at 1732628263170 2024-11-26T13:37:43,170 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17326281684493,,1732628174784.14cd7ab999b32e4f744910187c1c85c5. 2024-11-26T13:37:43,170 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 739931d97a34c012b15574f47d4b4161, disabling compactions & flushes 2024-11-26T13:37:43,170 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:37:43,170 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:37:43,170 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. after waiting 0 ms 2024-11-26T13:37:43,170 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:37:43,173 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system_bulk/739931d97a34c012b15574f47d4b4161/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-26T13:37:43,174 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:37:43,174 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:37:43,174 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 739931d97a34c012b15574f47d4b4161: Waiting for close lock at 1732628263170Running coprocessor pre-close hooks at 1732628263170Disabling compacts and flushes for region at 1732628263170Disabling writes for close at 1732628263170Writing region close event to WAL at 1732628263171 (+1 ms)Running coprocessor post-close hooks at 1732628263174 (+3 ms)Closed at 1732628263174 2024-11-26T13:37:43,174 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1732628180943.739931d97a34c012b15574f47d4b4161. 2024-11-26T13:37:43,174 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing be93584de20547dc73771fcf1a1b4605, disabling compactions & flushes 2024-11-26T13:37:43,174 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:43,245 INFO [regionserver/256175a329f8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T13:37:43,287 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/e8fb120c7dde4b2da06171453a176148 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/e8fb120c7dde4b2da06171453a176148 2024-11-26T13:37:43,287 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/meta/8f7127b1d85e40048c59ca9db679a4d3 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/8f7127b1d85e40048c59ca9db679a4d3 2024-11-26T13:37:43,300 INFO [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be93584de20547dc73771fcf1a1b4605/meta of be93584de20547dc73771fcf1a1b4605 into 8f7127b1d85e40048c59ca9db679a4d3(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T13:37:43,300 INFO [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be93584de20547dc73771fcf1a1b4605/session of be93584de20547dc73771fcf1a1b4605 into e8fb120c7dde4b2da06171453a176148(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T13:37:43,300 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be93584de20547dc73771fcf1a1b4605: 2024-11-26T13:37:43,300 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be93584de20547dc73771fcf1a1b4605: 2024-11-26T13:37:43,300 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:43,301 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:43,301 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. after waiting 0 ms 2024-11-26T13:37:43,301 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:43,301 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing be93584de20547dc73771fcf1a1b4605 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-11-26T13:37:43,303 INFO [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., storeName=be93584de20547dc73771fcf1a1b4605/session, priority=12, startTime=1732628262840; duration=0sec 2024-11-26T13:37:43,303 INFO [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., storeName=be93584de20547dc73771fcf1a1b4605/meta, priority=13, startTime=1732628262837; duration=0sec 2024-11-26T13:37:43,303 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T13:37:43,304 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T13:37:43,304 DEBUG [RS:0;256175a329f8:38807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be93584de20547dc73771fcf1a1b4605:meta 2024-11-26T13:37:43,304 DEBUG [RS:0;256175a329f8:38807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be93584de20547dc73771fcf1a1b4605:session 2024-11-26T13:37:43,304 INFO [RS:0;256175a329f8:38807 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T13:37:43,304 INFO [RS:0;256175a329f8:38807 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T13:37:43,304 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T13:37:43,304 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-26T13:37:43,305 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, be93584de20547dc73771fcf1a1b4605=backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., a62bf88ee852e800513fdadb466dea7e=ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e., 1d27e55c185bde71afb21ebc132e1a4f=test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f.} 2024-11-26T13:37:43,305 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T13:37:43,305 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T13:37:43,305 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T13:37:43,305 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T13:37:43,305 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T13:37:43,305 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d27e55c185bde71afb21ebc132e1a4f, a62bf88ee852e800513fdadb466dea7e, be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:43,305 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-11-26T13:37:43,306 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/929682fd71414d1998d0048b32c1259f is 554, key is session:backup_1732628247464/session:context/1732628263112/Put/seqid=0 2024-11-26T13:37:43,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741976_1152 (size=5655) 2024-11-26T13:37:43,328 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/info/f5d68b2dff764e3a99fb96b38d7fe0f0 is 159, key is ns2:test-17326281684491,,1732628170279.d333b64a51b935fe3679e64e25b2142a./info:regioninfo/1732628171467/Put/seqid=0 2024-11-26T13:37:43,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741977_1153 (size=14126) 2024-11-26T13:37:43,505 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d27e55c185bde71afb21ebc132e1a4f, a62bf88ee852e800513fdadb466dea7e, be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:43,705 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d27e55c185bde71afb21ebc132e1a4f, a62bf88ee852e800513fdadb466dea7e, be93584de20547dc73771fcf1a1b4605 2024-11-26T13:37:43,711 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/929682fd71414d1998d0048b32c1259f 2024-11-26T13:37:43,717 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/.tmp/session/929682fd71414d1998d0048b32c1259f as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/929682fd71414d1998d0048b32c1259f 2024-11-26T13:37:43,722 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/929682fd71414d1998d0048b32c1259f, entries=2, sequenceid=57, filesize=5.5 K 2024-11-26T13:37:43,723 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for be93584de20547dc73771fcf1a1b4605 in 422ms, sequenceid=57, compaction requested=false 2024-11-26T13:37:43,724 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125] to archive 2024-11-26T13:37:43,725 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T13:37:43,728 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/ef83da5ee1434703a7a27938ed98d745 2024-11-26T13:37:43,730 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/1e45e166134e4296afca96b815dd2b5c 2024-11-26T13:37:43,731 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/meta/d6ded4b85c3e44deb2d04621202ff125 2024-11-26T13:37:43,733 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/info/f5d68b2dff764e3a99fb96b38d7fe0f0 2024-11-26T13:37:43,737 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=256175a329f8:40825 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-26T13:37:43,738 WARN [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [ef83da5ee1434703a7a27938ed98d745=5914, 1e45e166134e4296afca96b815dd2b5c=5745, d6ded4b85c3e44deb2d04621202ff125=6646] 2024-11-26T13:37:43,738 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293] to archive 2024-11-26T13:37:43,739 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T13:37:43,741 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/d39591ef2cef4c0299ebddb16931dca4 2024-11-26T13:37:43,742 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/87d571180ca341c4a22fd473d47010cf 2024-11-26T13:37:43,744 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/b985f9302cfc4d61973fd5a8b948ccb0 2024-11-26T13:37:43,745 DEBUG [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293 to hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/archive/data/backup/system/be93584de20547dc73771fcf1a1b4605/session/84eb10e5568d48d381d0638dc24f6293 2024-11-26T13:37:43,745 WARN [StoreCloser-backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d39591ef2cef4c0299ebddb16931dca4=5049, 87d571180ca341c4a22fd473d47010cf=5535, b985f9302cfc4d61973fd5a8b948ccb0=6682, 84eb10e5568d48d381d0638dc24f6293=5928] 2024-11-26T13:37:43,749 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/backup/system/be93584de20547dc73771fcf1a1b4605/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-11-26T13:37:43,749 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:37:43,749 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:43,749 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for be93584de20547dc73771fcf1a1b4605: Waiting for close lock at 1732628263174Running coprocessor pre-close hooks at 1732628263174Disabling compacts and flushes for region at 1732628263174Disabling writes for close at 1732628263301 (+127 ms)Obtaining lock to block concurrent updates at 1732628263301Preparing flush snapshotting stores in be93584de20547dc73771fcf1a1b4605 at 1732628263301Finished memstore snapshotting backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1732628263301Flushing stores of backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. at 1732628263302 (+1 ms)Flushing be93584de20547dc73771fcf1a1b4605/session: creating writer at 1732628263302Flushing be93584de20547dc73771fcf1a1b4605/session: appending metadata at 1732628263305 (+3 ms)Flushing be93584de20547dc73771fcf1a1b4605/session: closing flushed file at 1732628263305Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d017d19: reopening flushed file at 1732628263717 (+412 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for be93584de20547dc73771fcf1a1b4605 in 422ms, sequenceid=57, compaction requested=false at 1732628263723 (+6 ms)Writing region close event to WAL at 1732628263746 (+23 ms)Running coprocessor post-close hooks at 1732628263749 (+3 ms)Closed at 1732628263749 2024-11-26T13:37:43,749 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1732628178770.be93584de20547dc73771fcf1a1b4605. 2024-11-26T13:37:43,750 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a62bf88ee852e800513fdadb466dea7e, disabling compactions & flushes 2024-11-26T13:37:43,750 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:37:43,750 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:37:43,750 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. after waiting 0 ms 2024-11-26T13:37:43,750 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:37:43,753 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/ns3/test-17326281684492/a62bf88ee852e800513fdadb466dea7e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-26T13:37:43,753 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:37:43,753 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:37:43,753 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a62bf88ee852e800513fdadb466dea7e: Waiting for close lock at 1732628263750Running coprocessor pre-close hooks at 1732628263750Disabling compacts and flushes for region at 1732628263750Disabling writes for close at 1732628263750Writing region close event to WAL at 1732628263750Running coprocessor post-close hooks at 1732628263753 (+3 ms)Closed at 1732628263753 2024-11-26T13:37:43,753 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17326281684492,,1732628172610.a62bf88ee852e800513fdadb466dea7e. 2024-11-26T13:37:43,754 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1d27e55c185bde71afb21ebc132e1a4f, disabling compactions & flushes 2024-11-26T13:37:43,754 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:37:43,754 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:37:43,754 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. after waiting 0 ms 2024-11-26T13:37:43,754 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:37:43,754 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1d27e55c185bde71afb21ebc132e1a4f 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-26T13:37:43,754 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/ns/0f713244e6834bc8acfccfb25fc91817 is 43, key is default/ns:d/1732628163798/Put/seqid=0 2024-11-26T13:37:43,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741978_1154 (size=5345) 2024-11-26T13:37:43,770 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f2/ec6546f4c1f8457499360fc0a01cd46f is 58, key is row-test-1732628168449-2-0/f2:q1/1732628242457/Put/seqid=0 2024-11-26T13:37:43,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741979_1155 (size=5464) 2024-11-26T13:37:43,905 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:37:44,048 INFO [regionserver/256175a329f8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T13:37:44,048 INFO [regionserver/256175a329f8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T13:37:44,106 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d27e55c185bde71afb21ebc132e1a4f 2024-11-26T13:37:44,159 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/ns/0f713244e6834bc8acfccfb25fc91817 2024-11-26T13:37:44,175 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f2/ec6546f4c1f8457499360fc0a01cd46f 2024-11-26T13:37:44,180 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/table/b8df43fecbeb47309c6ef7d21f221bd3 is 59, key is ns2:test-17326281684491/table:state/1732628171481/Put/seqid=0 2024-11-26T13:37:44,181 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/.tmp/f2/ec6546f4c1f8457499360fc0a01cd46f as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f2/ec6546f4c1f8457499360fc0a01cd46f 2024-11-26T13:37:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741980_1156 (size=5866) 2024-11-26T13:37:44,186 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/f2/ec6546f4c1f8457499360fc0a01cd46f, entries=7, sequenceid=348, filesize=5.3 K 2024-11-26T13:37:44,187 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 1d27e55c185bde71afb21ebc132e1a4f in 433ms, sequenceid=348, compaction requested=false 2024-11-26T13:37:44,192 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/default/test-1732628168449/1d27e55c185bde71afb21ebc132e1a4f/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-26T13:37:44,193 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:37:44,193 INFO [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:37:44,193 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1d27e55c185bde71afb21ebc132e1a4f: Waiting for close lock at 1732628263754Running coprocessor pre-close hooks at 1732628263754Disabling compacts and flushes for region at 1732628263754Disabling writes for close at 1732628263754Obtaining lock to block concurrent updates at 1732628263754Preparing flush snapshotting stores in 1d27e55c185bde71afb21ebc132e1a4f at 1732628263754Finished memstore snapshotting test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1732628263754Flushing stores of test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. at 1732628263754Flushing 1d27e55c185bde71afb21ebc132e1a4f/f2: creating writer at 1732628263755 (+1 ms)Flushing 1d27e55c185bde71afb21ebc132e1a4f/f2: appending metadata at 1732628263770 (+15 ms)Flushing 1d27e55c185bde71afb21ebc132e1a4f/f2: closing flushed file at 1732628263770Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41a33fbb: reopening flushed file at 1732628264180 (+410 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 1d27e55c185bde71afb21ebc132e1a4f in 433ms, sequenceid=348, compaction requested=false at 1732628264187 (+7 ms)Writing region close event to WAL at 1732628264189 (+2 ms)Running coprocessor post-close hooks at 1732628264192 (+3 ms)Closed at 1732628264193 (+1 ms) 2024-11-26T13:37:44,193 DEBUG [RS_CLOSE_REGION-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1732628168449,,1732628168942.1d27e55c185bde71afb21ebc132e1a4f. 2024-11-26T13:37:44,306 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T13:37:44,306 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T13:37:44,306 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T13:37:44,506 DEBUG [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T13:37:44,585 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/table/b8df43fecbeb47309c6ef7d21f221bd3 2024-11-26T13:37:44,591 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/info/f5d68b2dff764e3a99fb96b38d7fe0f0 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/info/f5d68b2dff764e3a99fb96b38d7fe0f0 2024-11-26T13:37:44,596 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/info/f5d68b2dff764e3a99fb96b38d7fe0f0, entries=69, sequenceid=59, filesize=13.8 K 2024-11-26T13:37:44,597 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/ns/0f713244e6834bc8acfccfb25fc91817 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/ns/0f713244e6834bc8acfccfb25fc91817 2024-11-26T13:37:44,601 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/ns/0f713244e6834bc8acfccfb25fc91817, entries=7, sequenceid=59, filesize=5.2 K 2024-11-26T13:37:44,602 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/.tmp/table/b8df43fecbeb47309c6ef7d21f221bd3 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/table/b8df43fecbeb47309c6ef7d21f221bd3 2024-11-26T13:37:44,607 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/table/b8df43fecbeb47309c6ef7d21f221bd3, entries=13, sequenceid=59, filesize=5.7 K 2024-11-26T13:37:44,608 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1303ms, sequenceid=59, compaction requested=false 2024-11-26T13:37:44,613 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-11-26T13:37:44,613 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-26T13:37:44,613 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T13:37:44,613 INFO [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T13:37:44,614 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732628263305Running coprocessor pre-close hooks at 1732628263305Disabling compacts and flushes for region at 1732628263305Disabling writes for close at 1732628263305Obtaining lock to block concurrent updates at 1732628263305Preparing flush snapshotting stores in 1588230740 at 1732628263305Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1732628263305Flushing stores of hbase:meta,,1.1588230740 at 1732628263306 (+1 ms)Flushing 1588230740/info: creating writer at 1732628263306Flushing 1588230740/info: appending metadata at 1732628263328 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732628263328Flushing 1588230740/ns: creating writer at 1732628263738 (+410 ms)Flushing 1588230740/ns: appending metadata at 1732628263754 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732628263754Flushing 1588230740/table: creating writer at 1732628264164 (+410 ms)Flushing 1588230740/table: appending metadata at 1732628264179 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732628264179Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32d4bb7a: reopening flushed file at 1732628264590 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39e00529: reopening flushed file at 1732628264596 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@145dbea3: reopening flushed file at 1732628264602 (+6 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1303ms, sequenceid=59, compaction requested=false at 1732628264608 (+6 ms)Writing region close event to WAL at 1732628264610 (+2 ms)Running coprocessor post-close hooks at 1732628264613 (+3 ms)Closed at 1732628264613 2024-11-26T13:37:44,614 DEBUG [RS_CLOSE_META-regionserver/256175a329f8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T13:37:44,706 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(976): stopping server 256175a329f8,38807,1732628159830; all regions closed. 2024-11-26T13:37:44,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741954_1130 (size=3260) 2024-11-26T13:37:44,710 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.meta.1732628251812.meta not finished, retry = 0 2024-11-26T13:37:44,816 DEBUG [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs 2024-11-26T13:37:44,816 INFO [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.meta:.meta(num 1732628251812) 2024-11-26T13:37:44,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741955_1131 (size=5275) 2024-11-26T13:37:44,818 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1.1732628251831 not finished, retry = 0 2024-11-26T13:37:44,922 DEBUG [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs 2024-11-26T13:37:44,922 INFO [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-1:(num 1732628251831) 2024-11-26T13:37:44,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741956_1132 (size=1037) 2024-11-26T13:37:44,924 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/WALs/256175a329f8,38807,1732628159830/256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0.1732628251849 not finished, retry = 0 2024-11-26T13:37:45,027 DEBUG [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/oldWALs 2024-11-26T13:37:45,027 INFO [RS:0;256175a329f8:38807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 256175a329f8%2C38807%2C1732628159830.256175a329f8%2C38807%2C1732628159830.regiongroup-0:(num 1732628251849) 2024-11-26T13:37:45,027 DEBUG [RS:0;256175a329f8:38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:45,027 INFO [RS:0;256175a329f8:38807 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T13:37:45,027 INFO [RS:0;256175a329f8:38807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T13:37:45,028 INFO [RS:0;256175a329f8:38807 {}] hbase.ChoreService(370): Chore service for: regionserver/256175a329f8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T13:37:45,028 INFO [RS:0;256175a329f8:38807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T13:37:45,028 INFO [regionserver/256175a329f8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T13:37:45,029 INFO [RS:0;256175a329f8:38807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38807 2024-11-26T13:37:45,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/256175a329f8,38807,1732628159830 2024-11-26T13:37:45,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-26T13:37:45,033 INFO [RS:0;256175a329f8:38807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T13:37:45,036 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [256175a329f8,38807,1732628159830] 2024-11-26T13:37:45,036 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-26T13:37:45,037 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/256175a329f8,38807,1732628159830 already deleted, retry=false 2024-11-26T13:37:45,038 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 256175a329f8,38807,1732628159830 expired; onlineServers=0 2024-11-26T13:37:45,038 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '256175a329f8,40825,1732628157903' ***** 2024-11-26T13:37:45,038 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T13:37:45,038 INFO [M:0;256175a329f8:40825 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-26T13:37:45,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-26T13:37:45,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-26T13:37:45,045 INFO [M:0;256175a329f8:40825 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@56e6bf8{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master} 2024-11-26T13:37:45,045 INFO [M:0;256175a329f8:40825 {}] server.AbstractConnector(383): Stopped ServerConnector@69cb2ebd{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-26T13:37:45,046 INFO [M:0;256175a329f8:40825 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T13:37:45,046 INFO [M:0;256175a329f8:40825 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5b460d1c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-26T13:37:45,046 INFO [M:0;256175a329f8:40825 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7ca98a11{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,STOPPED} 2024-11-26T13:37:45,046 INFO [M:0;256175a329f8:40825 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T13:37:45,046 INFO [M:0;256175a329f8:40825 {}] master.HMaster(1770): Stopping master jetty server 2024-11-26T13:37:45,047 INFO [M:0;256175a329f8:40825 {}] server.AbstractConnector(383): Stopped ServerConnector@39c3cce8{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-26T13:37:45,047 INFO [M:0;256175a329f8:40825 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T13:37:45,047 DEBUG [M:0;256175a329f8:40825 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T13:37:45,048 INFO [M:0;256175a329f8:40825 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-26T13:37:45,048 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T13:37:45,048 INFO [M:0;256175a329f8:40825 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;256175a329f8:40825. 2024-11-26T13:37:45,048 DEBUG [master/256175a329f8:0:becomeActiveMaster-HFileCleaner.small.0-1732628162884 {}] cleaner.HFileCleaner(306): Exit Thread[master/256175a329f8:0:becomeActiveMaster-HFileCleaner.small.0-1732628162884,5,FailOnTimeoutGroup] 2024-11-26T13:37:45,048 DEBUG [master/256175a329f8:0:becomeActiveMaster-HFileCleaner.large.0-1732628162880 {}] cleaner.HFileCleaner(306): Exit Thread[master/256175a329f8:0:becomeActiveMaster-HFileCleaner.large.0-1732628162880,5,FailOnTimeoutGroup] 2024-11-26T13:37:45,048 DEBUG [M:0;256175a329f8:40825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T13:37:45,048 DEBUG [M:0;256175a329f8:40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:45,048 DEBUG [M:0;256175a329f8:40825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T13:37:45,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T13:37:45,048 DEBUG [M:0;256175a329f8:40825 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T13:37:45,048 INFO [M:0;256175a329f8:40825 {}] hbase.ChoreService(370): Chore service for: master/256175a329f8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T13:37:45,048 INFO [M:0;256175a329f8:40825 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T13:37:45,049 DEBUG [M:0;256175a329f8:40825 {}] master.HMaster(1795): Stopping service threads 2024-11-26T13:37:45,049 INFO [M:0;256175a329f8:40825 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T13:37:45,049 INFO [M:0;256175a329f8:40825 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T13:37:45,049 INFO [M:0;256175a329f8:40825 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T13:37:45,050 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T13:37:45,050 DEBUG [M:0;256175a329f8:40825 {}] zookeeper.ZKUtil(347): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-26T13:37:45,050 WARN [M:0;256175a329f8:40825 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T13:37:45,051 INFO [M:0;256175a329f8:40825 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/.lastflushedseqids 2024-11-26T13:37:45,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741981_1157 (size=504) 2024-11-26T13:37:45,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T13:37:45,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x100340b4b270001, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T13:37:45,137 INFO [RS:0;256175a329f8:38807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T13:37:45,137 INFO [RS:0;256175a329f8:38807 {}] regionserver.HRegionServer(1031): Exiting; stopping=256175a329f8,38807,1732628159830; zookeeper connection closed. 2024-11-26T13:37:45,137 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3e6233e5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3e6233e5 2024-11-26T13:37:45,138 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T13:37:45,466 INFO [M:0;256175a329f8:40825 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T13:37:45,466 INFO [M:0;256175a329f8:40825 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T13:37:45,466 DEBUG [M:0;256175a329f8:40825 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T13:37:45,480 INFO [M:0;256175a329f8:40825 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T13:37:45,481 DEBUG [M:0;256175a329f8:40825 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T13:37:45,481 DEBUG [M:0;256175a329f8:40825 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T13:37:45,481 DEBUG [M:0;256175a329f8:40825 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T13:37:45,481 INFO [M:0;256175a329f8:40825 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=319.25 KB heapSize=375.36 KB 2024-11-26T13:37:45,497 DEBUG [M:0;256175a329f8:40825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/396fd6cb29724e4ea4abba6ede4d75d1 is 82, key is hbase:meta,,1/info:regioninfo/1732628163703/Put/seqid=0 2024-11-26T13:37:45,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741982_1158 (size=5672) 2024-11-26T13:37:45,902 INFO [M:0;256175a329f8:40825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/396fd6cb29724e4ea4abba6ede4d75d1 2024-11-26T13:37:45,924 DEBUG [M:0;256175a329f8:40825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bbaf42de1e94cd098f996cfe71f53ce is 2183, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1732628213515/Put/seqid=0 2024-11-26T13:37:45,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741983_1159 (size=26519) 2024-11-26T13:37:46,329 INFO [M:0;256175a329f8:40825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.70 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bbaf42de1e94cd098f996cfe71f53ce 2024-11-26T13:37:46,335 INFO [M:0;256175a329f8:40825 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6bbaf42de1e94cd098f996cfe71f53ce 2024-11-26T13:37:46,350 DEBUG [M:0;256175a329f8:40825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0d12d7bc5424c25b7f8a8dcec894127 is 69, key is 256175a329f8,38807,1732628159830/rs:state/1732628162973/Put/seqid=0 2024-11-26T13:37:46,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741984_1160 (size=5156) 2024-11-26T13:37:46,755 INFO [M:0;256175a329f8:40825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0d12d7bc5424c25b7f8a8dcec894127 2024-11-26T13:37:46,762 DEBUG [M:0;256175a329f8:40825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/396fd6cb29724e4ea4abba6ede4d75d1 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/396fd6cb29724e4ea4abba6ede4d75d1 2024-11-26T13:37:46,767 INFO [M:0;256175a329f8:40825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/396fd6cb29724e4ea4abba6ede4d75d1, entries=8, sequenceid=667, filesize=5.5 K 2024-11-26T13:37:46,768 DEBUG [M:0;256175a329f8:40825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bbaf42de1e94cd098f996cfe71f53ce as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6bbaf42de1e94cd098f996cfe71f53ce 2024-11-26T13:37:46,772 INFO [M:0;256175a329f8:40825 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6bbaf42de1e94cd098f996cfe71f53ce 2024-11-26T13:37:46,772 INFO [M:0;256175a329f8:40825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6bbaf42de1e94cd098f996cfe71f53ce, entries=69, sequenceid=667, filesize=25.9 K 2024-11-26T13:37:46,773 DEBUG [M:0;256175a329f8:40825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0d12d7bc5424c25b7f8a8dcec894127 as hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0d12d7bc5424c25b7f8a8dcec894127 2024-11-26T13:37:46,777 INFO [M:0;256175a329f8:40825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36063/user/jenkins/test-data/64f81304-322d-c23c-2a48-28e2fca9bba9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0d12d7bc5424c25b7f8a8dcec894127, entries=1, sequenceid=667, filesize=5.0 K 2024-11-26T13:37:46,778 INFO [M:0;256175a329f8:40825 {}] regionserver.HRegion(3140): Finished flush of dataSize ~319.25 KB/326915, heapSize ~375.06 KB/384064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=667, compaction requested=false 2024-11-26T13:37:46,780 INFO [M:0;256175a329f8:40825 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T13:37:46,780 DEBUG [M:0;256175a329f8:40825 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732628265466Disabling compacts and flushes for region at 1732628265466Disabling writes for close at 1732628265481 (+15 ms)Obtaining lock to block concurrent updates at 1732628265481Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732628265481Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326915, getHeapSize=384304, getOffHeapSize=0, getCellsCount=746 at 1732628265481Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732628265482 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732628265482Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732628265497 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732628265497Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732628265907 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732628265923 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732628265923Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732628266335 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732628266350 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732628266350Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d1d7db: reopening flushed file at 1732628266761 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@564fb5f6: reopening flushed file at 1732628266767 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aacd2ee: reopening flushed file at 1732628266772 (+5 ms)Finished flush of dataSize ~319.25 KB/326915, heapSize ~375.06 KB/384064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=667, compaction requested=false at 1732628266778 (+6 ms)Writing region close event to WAL at 1732628266780 (+2 ms)Closed at 1732628266780 2024-11-26T13:37:46,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46229 is added to blk_1073741830_1006 (size=371005) 2024-11-26T13:37:46,783 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T13:37:46,784 INFO [M:0;256175a329f8:40825 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T13:37:46,784 INFO [M:0;256175a329f8:40825 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-26T13:37:46,784 INFO [M:0;256175a329f8:40825 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40825 2024-11-26T13:37:46,784 INFO [M:0;256175a329f8:40825 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T13:37:46,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T13:37:46,886 INFO [M:0;256175a329f8:40825 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T13:37:46,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40825-0x100340b4b270000, quorum=127.0.0.1:62595, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T13:37:46,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6628e1d0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T13:37:46,896 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d5d982f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T13:37:46,896 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T13:37:46,896 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6be2f62b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T13:37:46,896 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b9b7476{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,STOPPED} 2024-11-26T13:37:46,899 WARN [BP-650579929-172.17.0.2-1732628154247 heartbeating to localhost/127.0.0.1:36063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T13:37:46,899 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T13:37:46,899 WARN [BP-650579929-172.17.0.2-1732628154247 heartbeating to localhost/127.0.0.1:36063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-650579929-172.17.0.2-1732628154247 (Datanode Uuid 9ccdf160-6c18-4a5c-9556-fc6dc2fa79c8) service to localhost/127.0.0.1:36063 2024-11-26T13:37:46,899 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T13:37:46,900 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/data/data1/current/BP-650579929-172.17.0.2-1732628154247 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T13:37:46,901 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/cluster_df6735c7-fb88-c88c-31ac-d041c93b25dd/data/data2/current/BP-650579929-172.17.0.2-1732628154247 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T13:37:46,901 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T13:37:46,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6df1a11{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T13:37:46,908 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34c8bc74{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T13:37:46,908 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T13:37:46,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17280962{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T13:37:46,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71f46724{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,STOPPED} 2024-11-26T13:37:46,917 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T13:37:46,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T13:37:46,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-26T13:37:46,974 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49fbadf3{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-26T13:37:46,975 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@67020615{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T13:37:46,975 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T13:37:46,975 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42fc61ef{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-26T13:37:46,975 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54c34ba5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,STOPPED} 2024-11-26T13:37:46,989 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@128c7511{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-26T13:37:46,990 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d8527a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T13:37:46,990 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T13:37:46,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c406c80{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-26T13:37:46,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3476a55f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,STOPPED} 2024-11-26T13:37:46,995 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-26T13:37:46,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@414bbc4c{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-26T13:37:46,996 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@382d96c0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T13:37:46,996 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T13:37:46,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11a358cb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-26T13:37:46,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@339da2c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,STOPPED} 2024-11-26T13:37:46,999 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-26T13:37:47,003 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-26T13:37:47,003 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-26T13:37:47,004 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-26T13:37:47,007 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3a4583c9{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-26T13:37:47,008 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6d50a15e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T13:37:47,008 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T13:37:47,008 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5bc8e33d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-26T13:37:47,008 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@485c5160{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/8fb9119b-5fb4-029d-a37b-b506c1432d23/hadoop.log.dir/,STOPPED} 2024-11-26T13:37:47,010 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-26T13:37:47,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped