2024-11-24 02:34:18,983 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-24 02:34:19,006 main DEBUG Took 0.019708 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 02:34:19,007 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 02:34:19,007 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 02:34:19,010 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 02:34:19,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,037 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 02:34:19,056 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,058 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,059 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,059 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,060 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,060 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,061 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,062 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,063 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,063 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,064 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,065 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,065 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,066 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,066 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,067 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,068 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,069 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,070 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,071 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:34:19,073 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,073 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 02:34:19,075 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:34:19,077 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 02:34:19,080 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 02:34:19,081 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 02:34:19,083 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 02:34:19,083 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 02:34:19,098 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 02:34:19,104 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 02:34:19,107 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 02:34:19,108 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 02:34:19,108 main DEBUG createAppenders(={Console}) 2024-11-24 02:34:19,109 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-24 02:34:19,110 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-24 02:34:19,110 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-24 02:34:19,111 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 02:34:19,111 main DEBUG OutputStream closed 2024-11-24 02:34:19,112 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 02:34:19,112 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 02:34:19,113 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-24 02:34:19,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 02:34:19,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 02:34:19,228 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 02:34:19,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 02:34:19,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 02:34:19,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 02:34:19,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 02:34:19,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 02:34:19,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 02:34:19,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 02:34:19,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 02:34:19,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 02:34:19,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 02:34:19,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 02:34:19,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 02:34:19,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 02:34:19,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 02:34:19,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 02:34:19,241 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 02:34:19,241 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-24 02:34:19,242 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 02:34:19,243 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-24T02:34:19,357 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-24 02:34:19,362 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 02:34:19,363 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T02:34:19,952 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e 2024-11-24T02:34:19,958 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-24T02:34:19,961 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:34:19,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T02:34:19,987 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417, deleteOnExit=true 2024-11-24T02:34:19,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T02:34:19,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/test.cache.data in system properties and HBase conf 2024-11-24T02:34:19,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T02:34:19,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir in system properties and HBase conf 2024-11-24T02:34:19,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T02:34:19,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T02:34:19,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T02:34:20,142 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T02:34:20,306 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T02:34:20,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T02:34:20,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T02:34:20,317 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T02:34:20,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T02:34:20,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T02:34:20,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T02:34:20,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T02:34:20,321 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T02:34:20,321 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T02:34:20,322 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/nfs.dump.dir in system properties and HBase conf 2024-11-24T02:34:20,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/java.io.tmpdir in system properties and HBase conf 2024-11-24T02:34:20,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T02:34:20,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T02:34:20,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T02:34:22,032 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T02:34:22,141 INFO [Time-limited test {}] log.Log(170): Logging initialized @4359ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T02:34:22,257 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:22,388 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:34:22,437 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:34:22,437 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:34:22,440 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:34:22,481 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:22,498 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71f46724{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:34:22,500 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17280962{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T02:34:22,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6df1a11{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/java.io.tmpdir/jetty-localhost-44263-hadoop-hdfs-3_4_1-tests_jar-_-any-11821516898272845312/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T02:34:22,819 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34c8bc74{HTTP/1.1, (http/1.1)}{localhost:44263} 2024-11-24T02:34:22,819 INFO [Time-limited test {}] server.Server(415): Started @5039ms 2024-11-24T02:34:23,502 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:23,511 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:34:23,512 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:34:23,512 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:34:23,512 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T02:34:23,513 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b9b7476{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:34:23,514 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6be2f62b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T02:34:23,627 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6628e1d0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/java.io.tmpdir/jetty-localhost-45529-hadoop-hdfs-3_4_1-tests_jar-_-any-13028367781104281639/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T02:34:23,628 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d5d982f{HTTP/1.1, (http/1.1)}{localhost:45529} 2024-11-24T02:34:23,629 INFO [Time-limited test {}] server.Server(415): Started @5848ms 2024-11-24T02:34:23,692 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T02:34:25,052 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/data/data2/current/BP-1481479676-172.17.0.2-1732415661183/current, will proceed with Du for space computation calculation, 2024-11-24T02:34:25,052 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/data/data1/current/BP-1481479676-172.17.0.2-1732415661183/current, will proceed with Du for space computation calculation, 2024-11-24T02:34:25,081 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T02:34:25,133 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc45af7ceb0f42ecf with lease ID 0x1064dd9397c91e39: Processing first storage report for DS-4508af20-af3c-4cc1-a699-29e9b8c67228 from datanode DatanodeRegistration(127.0.0.1:34653, datanodeUuid=fba3894c-5686-4c5f-8aec-1ea8dccd98b4, infoPort=43173, infoSecurePort=0, ipcPort=39205, storageInfo=lv=-57;cid=testClusterID;nsid=1512028441;c=1732415661183) 2024-11-24T02:34:25,134 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc45af7ceb0f42ecf with lease ID 0x1064dd9397c91e39: from storage DS-4508af20-af3c-4cc1-a699-29e9b8c67228 node DatanodeRegistration(127.0.0.1:34653, datanodeUuid=fba3894c-5686-4c5f-8aec-1ea8dccd98b4, infoPort=43173, infoSecurePort=0, ipcPort=39205, storageInfo=lv=-57;cid=testClusterID;nsid=1512028441;c=1732415661183), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T02:34:25,134 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc45af7ceb0f42ecf with lease ID 0x1064dd9397c91e39: Processing first storage report for DS-52323ac4-8074-43c2-8ff0-6bb607175318 from datanode DatanodeRegistration(127.0.0.1:34653, datanodeUuid=fba3894c-5686-4c5f-8aec-1ea8dccd98b4, infoPort=43173, infoSecurePort=0, ipcPort=39205, storageInfo=lv=-57;cid=testClusterID;nsid=1512028441;c=1732415661183) 2024-11-24T02:34:25,134 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc45af7ceb0f42ecf with lease ID 0x1064dd9397c91e39: from storage DS-52323ac4-8074-43c2-8ff0-6bb607175318 node DatanodeRegistration(127.0.0.1:34653, datanodeUuid=fba3894c-5686-4c5f-8aec-1ea8dccd98b4, infoPort=43173, infoSecurePort=0, ipcPort=39205, storageInfo=lv=-57;cid=testClusterID;nsid=1512028441;c=1732415661183), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T02:34:25,163 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e 2024-11-24T02:34:25,272 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/zookeeper_0, clientPort=58752, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T02:34:25,283 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58752 2024-11-24T02:34:25,298 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:34:25,303 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:34:25,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741825_1001 (size=7) 2024-11-24T02:34:25,976 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 with version=8 2024-11-24T02:34:25,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging 2024-11-24T02:34:25,990 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-24T02:34:25,991 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-24T02:34:25,991 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-24T02:34:25,991 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-24T02:34:26,089 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T02:34:26,343 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a67e7d17e593:0 server-side Connection retries=45 2024-11-24T02:34:26,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:34:26,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T02:34:26,358 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T02:34:26,359 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:34:26,359 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T02:34:26,521 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T02:34:26,606 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T02:34:26,618 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T02:34:26,624 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T02:34:26,652 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 151209 (auto-detected) 2024-11-24T02:34:26,653 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T02:34:26,677 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41423 2024-11-24T02:34:26,702 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41423 connecting to ZooKeeper ensemble=127.0.0.1:58752 2024-11-24T02:34:26,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414230x0, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T02:34:26,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41423-0x1016bfac7200000 connected 2024-11-24T02:34:26,860 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:34:26,863 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:34:26,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-24T02:34:26,894 INFO [Time-limited test {}] log.Log(170): Logging initialized @9114ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T02:34:26,947 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-24T02:34:26,947 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-24T02:34:26,948 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-24T02:34:26,948 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-24T02:34:26,949 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-24T02:34:26,950 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-24T02:34:26,951 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-24T02:34:26,951 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-24T02:34:26,953 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-24T02:34:27,019 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 39187 2024-11-24T02:34:27,020 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-24T02:34:27,034 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T02:34:27,035 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@613be4cc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:34:27,036 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T02:34:27,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@e90fde3{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-24T02:34:27,075 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-24T02:34:27,082 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:34:27,083 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:34:27,084 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:34:27,087 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T02:34:27,789 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@6b147c85{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master} 2024-11-24T02:34:27,794 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3bf75574{HTTP/1.1, (http/1.1)}{0.0.0.0:39187} 2024-11-24T02:34:27,795 INFO [Time-limited test {}] server.Server(415): Started @10015ms 2024-11-24T02:34:27,799 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725, hbase.cluster.distributed=false 2024-11-24T02:34:27,831 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-24T02:34:27,837 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41423 2024-11-24T02:34:27,837 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41423 2024-11-24T02:34:27,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41423 2024-11-24T02:34:27,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41423 2024-11-24T02:34:27,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41423 2024-11-24T02:34:27,972 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a67e7d17e593:0 server-side Connection retries=45 2024-11-24T02:34:27,973 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:34:27,973 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T02:34:27,974 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T02:34:27,974 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:34:27,974 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T02:34:27,977 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T02:34:27,979 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T02:34:27,980 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38807 2024-11-24T02:34:27,982 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38807 connecting to ZooKeeper ensemble=127.0.0.1:58752 2024-11-24T02:34:27,984 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:34:27,986 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:34:28,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388070x0, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T02:34:28,032 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388070x0, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-24T02:34:28,032 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38807-0x1016bfac7200001 connected 2024-11-24T02:34:28,035 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-24T02:34:28,035 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-24T02:34:28,038 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-24T02:34:28,038 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-24T02:34:28,038 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-24T02:34:28,039 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-24T02:34:28,039 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-24T02:34:28,039 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-24T02:34:28,040 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-24T02:34:28,042 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 39827 2024-11-24T02:34:28,042 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-24T02:34:28,044 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T02:34:28,044 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@65eb8eac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:34:28,045 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T02:34:28,045 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@160a6036{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-24T02:34:28,060 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-24T02:34:28,062 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:34:28,062 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:34:28,062 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T02:34:28,063 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-24T02:34:28,066 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@72979367{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-24T02:34:28,067 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1cdf8417{HTTP/1.1, (http/1.1)}{0.0.0.0:39827} 2024-11-24T02:34:28,067 INFO [Time-limited test {}] server.Server(415): Started @10287ms 2024-11-24T02:34:28,071 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T02:34:28,079 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T02:34:28,082 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-24T02:34:28,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-24T02:34:28,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38807 2024-11-24T02:34:28,089 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38807 2024-11-24T02:34:28,089 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38807 2024-11-24T02:34:28,091 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38807 2024-11-24T02:34:28,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38807 2024-11-24T02:34:28,115 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a67e7d17e593:41423 2024-11-24T02:34:28,118 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-24T02:34:28,127 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@3007dbc3{HTTP/1.1, (http/1.1)}{0.0.0.0:46689} 2024-11-24T02:34:28,127 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] server.Server(415): Started @10347ms 2024-11-24T02:34:28,128 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/a67e7d17e593,41423,1732415666135 2024-11-24T02:34:28,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-24T02:34:28,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-24T02:34:28,161 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/a67e7d17e593,41423,1732415666135 2024-11-24T02:34:28,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:28,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-24T02:34:28,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:28,192 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-24T02:34:28,193 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/a67e7d17e593,41423,1732415666135 from backup master directory 2024-11-24T02:34:28,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/a67e7d17e593,41423,1732415666135 2024-11-24T02:34:28,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-24T02:34:28,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-24T02:34:28,209 WARN [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T02:34:28,209 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a67e7d17e593,41423,1732415666135 2024-11-24T02:34:28,211 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T02:34:28,212 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T02:34:28,278 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase.id] with ID: ec8b847e-5725-4935-9e6f-e811370a6839 2024-11-24T02:34:28,278 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.tmp/hbase.id 2024-11-24T02:34:28,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741826_1002 (size=42) 2024-11-24T02:34:28,694 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.tmp/hbase.id]:[hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase.id] 2024-11-24T02:34:28,748 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:34:28,754 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T02:34:28,776 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-24T02:34:28,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:28,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:28,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741827_1003 (size=196) 2024-11-24T02:34:29,275 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:34:29,277 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T02:34:29,284 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-24T02:34:29,286 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-24T02:34:29,296 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:34:29,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741828_1004 (size=1189) 2024-11-24T02:34:29,745 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store 2024-11-24T02:34:29,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741829_1005 (size=34) 2024-11-24T02:34:30,170 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T02:34:30,173 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:30,175 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T02:34:30,175 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:34:30,175 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:34:30,176 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T02:34:30,177 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:34:30,177 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:34:30,178 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732415670174Disabling compacts and flushes for region at 1732415670174Disabling writes for close at 1732415670177 (+3 ms)Writing region close event to WAL at 1732415670177Closed at 1732415670177 2024-11-24T02:34:30,180 WARN [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/.initializing 2024-11-24T02:34:30,181 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/WALs/a67e7d17e593,41423,1732415666135 2024-11-24T02:34:30,185 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:34:30,190 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor a67e7d17e593,41423,1732415666135.regiongroup-0 2024-11-24T02:34:30,206 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C41423%2C1732415666135.a67e7d17e593%2C41423%2C1732415666135.regiongroup-0, suffix=, logDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/WALs/a67e7d17e593,41423,1732415666135, archiveDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/oldWALs, maxLogs=10 2024-11-24T02:34:30,229 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/WALs/a67e7d17e593,41423,1732415666135/a67e7d17e593%2C41423%2C1732415666135.a67e7d17e593%2C41423%2C1732415666135.regiongroup-0.1732415670211, exclude list is [], retry=0 2024-11-24T02:34:30,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:34:30,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T02:34:30,294 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/WALs/a67e7d17e593,41423,1732415666135/a67e7d17e593%2C41423%2C1732415666135.a67e7d17e593%2C41423%2C1732415666135.regiongroup-0.1732415670211 2024-11-24T02:34:30,295 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:34:30,295 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:30,296 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:30,300 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,301 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T02:34:30,385 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:30,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:30,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T02:34:30,393 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:30,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:30,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,398 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T02:34:30,399 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:30,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:30,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,404 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T02:34:30,405 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:30,406 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:30,407 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,410 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,412 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,421 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,421 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,426 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T02:34:30,431 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:34:30,437 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:34:30,439 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9623683840, jitterRate=-0.10372459888458252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T02:34:30,449 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732415670323Initializing all the Stores at 1732415670325 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732415670326 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415670329 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415670330 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415670330Cleaning up temporary data from old regions at 1732415670422 (+92 ms)Region opened successfully at 1732415670449 (+27 ms) 2024-11-24T02:34:30,451 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T02:34:30,486 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b210b1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=a67e7d17e593/172.17.0.2:0 2024-11-24T02:34:30,525 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T02:34:30,536 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T02:34:30,536 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T02:34:30,539 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T02:34:30,541 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T02:34:30,546 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T02:34:30,546 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T02:34:30,571 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T02:34:30,582 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-24T02:34:30,654 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-24T02:34:30,658 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T02:34:30,660 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-24T02:34:30,671 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-24T02:34:30,674 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T02:34:30,678 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-24T02:34:30,688 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-24T02:34:30,690 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-24T02:34:30,704 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-24T02:34:30,734 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T02:34:30,749 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-24T02:34:30,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-24T02:34:30,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:30,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-24T02:34:30,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:30,795 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a67e7d17e593,41423,1732415666135, sessionid=0x1016bfac7200000, setting cluster-up flag (Was=false) 2024-11-24T02:34:30,806 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-24T02:34:30,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:30,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:30,888 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-24T02:34:30,890 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a67e7d17e593,41423,1732415666135 2024-11-24T02:34:30,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:30,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:30,971 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-24T02:34:30,973 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a67e7d17e593,41423,1732415666135 2024-11-24T02:34:30,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:30,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:31,024 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-24T02:34:31,026 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a67e7d17e593,41423,1732415666135 2024-11-24T02:34:31,033 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T02:34:31,100 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(746): ClusterId : ec8b847e-5725-4935-9e6f-e811370a6839 2024-11-24T02:34:31,101 INFO [RS:0;a67e7d17e593:38807 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-24T02:34:31,104 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T02:34:31,131 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T02:34:31,136 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T02:34:31,137 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-24T02:34:31,145 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T02:34:31,149 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-24T02:34:31,150 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T02:34:31,154 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T02:34:31,159 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T02:34:31,159 DEBUG [RS:0;a67e7d17e593:38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd89674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=a67e7d17e593/172.17.0.2:0 2024-11-24T02:34:31,161 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a67e7d17e593,41423,1732415666135 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T02:34:31,169 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a67e7d17e593:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:34:31,169 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a67e7d17e593:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:34:31,170 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a67e7d17e593:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:34:31,170 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a67e7d17e593:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:34:31,170 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a67e7d17e593:0, corePoolSize=10, maxPoolSize=10 2024-11-24T02:34:31,170 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,170 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a67e7d17e593:0, corePoolSize=2, maxPoolSize=2 2024-11-24T02:34:31,170 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,177 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a67e7d17e593:38807 2024-11-24T02:34:31,180 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T02:34:31,181 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T02:34:31,181 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T02:34:31,184 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67e7d17e593,41423,1732415666135 with port=38807, startcode=1732415667927 2024-11-24T02:34:31,189 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:34:31,190 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T02:34:31,191 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732415701191 2024-11-24T02:34:31,193 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T02:34:31,194 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T02:34:31,198 DEBUG [RS:0;a67e7d17e593:38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T02:34:31,199 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T02:34:31,200 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-24T02:34:31,200 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:31,200 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T02:34:31,201 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T02:34:31,200 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T02:34:31,201 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T02:34:31,202 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,215 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T02:34:31,216 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T02:34:31,216 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T02:34:31,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741831_1007 (size=1321) 2024-11-24T02:34:31,233 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ccf4d8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:31,234 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:34:31,238 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:34:31,250 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:34:31,256 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47591, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T02:34:31,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:34:31,262 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:34:31,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:34:31,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a88f0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:31,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:34:31,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:34:31,272 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:34:31,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e5fbe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:31,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:31,277 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-24T02:34:31,279 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T02:34:31,280 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T02:34:31,286 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.large.0-1732415671282,5,FailOnTimeoutGroup] 2024-11-24T02:34:31,287 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.small.0-1732415671287,5,FailOnTimeoutGroup] 2024-11-24T02:34:31,287 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,288 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T02:34:31,289 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,289 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,290 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T02:34:31,290 WARN [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T02:34:31,391 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67e7d17e593,41423,1732415666135 with port=38807, startcode=1732415667927 2024-11-24T02:34:31,394 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41423 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a67e7d17e593,38807,1732415667927 2024-11-24T02:34:31,396 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41423 {}] master.ServerManager(517): Registering regionserver=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:31,405 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:31,405 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40215 2024-11-24T02:34:31,405 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=39187 2024-11-24T02:34:31,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-24T02:34:31,456 DEBUG [RS:0;a67e7d17e593:38807 {}] zookeeper.ZKUtil(111): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rs/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:31,456 WARN [RS:0;a67e7d17e593:38807 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T02:34:31,457 INFO [RS:0;a67e7d17e593:38807 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-24T02:34:31,457 INFO [RS:0;a67e7d17e593:38807 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-24T02:34:31,457 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:31,458 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a67e7d17e593,38807,1732415667927] 2024-11-24T02:34:31,481 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T02:34:31,499 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T02:34:31,503 INFO [RS:0;a67e7d17e593:38807 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T02:34:31,503 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,504 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T02:34:31,509 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T02:34:31,510 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,510 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,510 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0, corePoolSize=2, maxPoolSize=2 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,511 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,512 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:34:31,512 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0, corePoolSize=3, maxPoolSize=3 2024-11-24T02:34:31,512 DEBUG [RS:0;a67e7d17e593:38807 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a67e7d17e593:0, corePoolSize=3, maxPoolSize=3 2024-11-24T02:34:31,513 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,514 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,514 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,514 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,514 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,514 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,38807,1732415667927-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T02:34:31,532 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T02:34:31,534 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,38807,1732415667927-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,534 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,534 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.Replication(171): a67e7d17e593,38807,1732415667927 started 2024-11-24T02:34:31,553 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:31,553 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1482): Serving as a67e7d17e593,38807,1732415667927, RpcServer on a67e7d17e593/172.17.0.2:38807, sessionid=0x1016bfac7200001 2024-11-24T02:34:31,554 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T02:34:31,554 DEBUG [RS:0;a67e7d17e593:38807 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a67e7d17e593,38807,1732415667927 2024-11-24T02:34:31,554 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,38807,1732415667927' 2024-11-24T02:34:31,554 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-24T02:34:31,555 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-24T02:34:31,556 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T02:34:31,556 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-24T02:34:31,556 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,38807,1732415667927' 2024-11-24T02:34:31,556 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:34:31,557 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:34:31,557 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-24T02:34:31,557 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-24T02:34:31,557 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T02:34:31,558 DEBUG [RS:0;a67e7d17e593:38807 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a67e7d17e593,38807,1732415667927 2024-11-24T02:34:31,558 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,38807,1732415667927' 2024-11-24T02:34:31,558 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-24T02:34:31,558 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-24T02:34:31,559 DEBUG [RS:0;a67e7d17e593:38807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T02:34:31,559 INFO [RS:0;a67e7d17e593:38807 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T02:34:31,559 INFO [RS:0;a67e7d17e593:38807 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T02:34:31,631 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T02:34:31,631 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:31,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741832_1008 (size=32) 2024-11-24T02:34:31,665 INFO [RS:0;a67e7d17e593:38807 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:34:31,665 INFO [RS:0;a67e7d17e593:38807 {}] monitor.StreamSlowMonitor(122): New stream slow monitor a67e7d17e593,38807,1732415667927.regiongroup-0 2024-11-24T02:34:31,669 INFO [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0, suffix=, logDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927, archiveDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs, maxLogs=32 2024-11-24T02:34:31,691 DEBUG [RS:0;a67e7d17e593:38807 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415671672, exclude list is [], retry=0 2024-11-24T02:34:31,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:34:31,700 INFO [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415671672 2024-11-24T02:34:31,701 DEBUG [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:34:32,043 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:32,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T02:34:32,049 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T02:34:32,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:32,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:32,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T02:34:32,053 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T02:34:32,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:32,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:32,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T02:34:32,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T02:34:32,058 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:32,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:32,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T02:34:32,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T02:34:32,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:32,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:32,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T02:34:32,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740 2024-11-24T02:34:32,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740 2024-11-24T02:34:32,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T02:34:32,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T02:34:32,073 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T02:34:32,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T02:34:32,079 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:34:32,081 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10071808160, jitterRate=-0.06198976933956146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T02:34:32,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732415672044Initializing all the Stores at 1732415672046 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732415672046Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732415672046Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415672046Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732415672046Cleaning up temporary data from old regions at 1732415672071 (+25 ms)Region opened successfully at 1732415672084 (+13 ms) 2024-11-24T02:34:32,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T02:34:32,085 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T02:34:32,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T02:34:32,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T02:34:32,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T02:34:32,087 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T02:34:32,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732415672085Disabling compacts and flushes for region at 1732415672085Disabling writes for close at 1732415672085Writing region close event to WAL at 1732415672087 (+2 ms)Closed at 1732415672087 2024-11-24T02:34:32,091 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:34:32,091 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T02:34:32,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T02:34:32,103 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T02:34:32,107 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T02:34:32,259 DEBUG [a67e7d17e593:41423 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T02:34:32,273 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:32,281 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a67e7d17e593,38807,1732415667927, state=OPENING 2024-11-24T02:34:32,299 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T02:34:32,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:32,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:34:32,308 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-24T02:34:32,308 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-24T02:34:32,310 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T02:34:32,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:32,486 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T02:34:32,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54877, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T02:34:32,498 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T02:34:32,499 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-24T02:34:32,499 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-24T02:34:32,499 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:34:32,500 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T02:34:32,503 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C38807%2C1732415667927.meta, suffix=.meta, logDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927, archiveDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs, maxLogs=32 2024-11-24T02:34:32,518 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415672505.meta, exclude list is [], retry=0 2024-11-24T02:34:32,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:34:32,527 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415672505.meta 2024-11-24T02:34:32,527 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:34:32,528 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:32,532 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:34:32,534 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T02:34:32,537 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T02:34:32,537 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T02:34:32,542 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T02:34:32,543 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:32,543 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T02:34:32,544 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T02:34:32,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T02:34:32,551 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T02:34:32,551 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:32,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:32,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T02:34:32,555 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T02:34:32,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:32,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:32,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T02:34:32,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T02:34:32,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:32,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:32,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T02:34:32,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T02:34:32,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:32,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:34:32,564 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T02:34:32,568 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740 2024-11-24T02:34:32,572 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740 2024-11-24T02:34:32,576 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T02:34:32,576 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T02:34:32,577 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T02:34:32,582 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T02:34:32,585 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11114154880, jitterRate=0.03508633375167847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T02:34:32,585 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T02:34:32,590 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732415672544Writing region info on filesystem at 1732415672545 (+1 ms)Initializing all the Stores at 1732415672547 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732415672547Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732415672548 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415672548Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732415672548Cleaning up temporary data from old regions at 1732415672576 (+28 ms)Running coprocessor post-open hooks at 1732415672585 (+9 ms)Region opened successfully at 1732415672590 (+5 ms) 2024-11-24T02:34:32,604 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732415672478 2024-11-24T02:34:32,618 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T02:34:32,619 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T02:34:32,620 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:32,623 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a67e7d17e593,38807,1732415667927, state=OPEN 2024-11-24T02:34:32,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-24T02:34:32,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-24T02:34:32,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-24T02:34:32,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-24T02:34:32,641 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:32,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T02:34:32,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a67e7d17e593,38807,1732415667927 in 329 msec 2024-11-24T02:34:32,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T02:34:32,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 554 msec 2024-11-24T02:34:32,662 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:34:32,662 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T02:34:32,683 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:34:32,685 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:34:32,710 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:34:32,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:34:32,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6740 sec 2024-11-24T02:34:32,751 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732415672751, completionTime=-1 2024-11-24T02:34:32,755 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T02:34:32,755 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T02:34:32,794 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T02:34:32,794 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732415732794 2024-11-24T02:34:32,795 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732415792795 2024-11-24T02:34:32,795 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 39 msec 2024-11-24T02:34:32,798 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,41423,1732415666135-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:32,799 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,41423,1732415666135-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:32,799 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,41423,1732415666135-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:32,801 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a67e7d17e593:41423, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:32,802 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:32,803 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:32,811 DEBUG [master/a67e7d17e593:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T02:34:32,851 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.642sec 2024-11-24T02:34:32,853 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T02:34:32,856 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T02:34:32,857 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T02:34:32,858 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T02:34:32,858 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T02:34:32,859 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,41423,1732415666135-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T02:34:32,860 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,41423,1732415666135-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T02:34:32,878 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T02:34:32,878 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T02:34:32,879 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,41423,1732415666135-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:34:32,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3afe73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:32,906 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T02:34:32,906 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T02:34:32,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:34:32,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:34:32,918 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:34:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:34:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:34:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57edd2d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:34:32,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:34:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:32,927 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:34:32,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ccfcc33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:32,929 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:34:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:34:32,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:34:32,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:34:32,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a67e7d17e593,41423,1732415666135 2024-11-24T02:34:32,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-24T02:34:32,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/test.cache.data in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T02:34:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T02:34:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T02:34:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T02:34:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T02:34:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/nfs.dump.dir in system properties and HBase conf 2024-11-24T02:34:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/java.io.tmpdir in system properties and HBase conf 2024-11-24T02:34:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T02:34:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T02:34:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T02:34:33,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741835_1011 (size=592039) 2024-11-24T02:34:33,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741836_1012 (size=1663647) 2024-11-24T02:34:34,969 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:35,065 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:35,255 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:34:35,255 WARN [Thread-251 {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-24T02:34:35,256 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:34:35,269 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:34:35,269 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:34:35,269 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:34:35,269 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:34:35,269 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:34:35,269 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:34:35,270 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32dc3d85{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:34:35,270 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22311523{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-24T02:34:35,272 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:35,280 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@349f1fd6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:34:35,281 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6f34157b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-24T02:34:35,414 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2024-11-24T02:34:35,414 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2024-11-24T02:34:35,414 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-24T02:34:35,417 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-24T02:34:35,472 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T02:34:35,523 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T02:34:35,811 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2024-11-24T02:34:35,830 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@33bd426e{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/java.io.tmpdir/jetty-localhost-43613-hadoop-yarn-common-3_4_1_jar-_-any-3117449302599032274/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-24T02:34:35,830 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5018e614{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/java.io.tmpdir/jetty-localhost-37507-hadoop-yarn-common-3_4_1_jar-_-any-3540661132594009546/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-24T02:34:35,831 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@423a9db4{HTTP/1.1, (http/1.1)}{localhost:43613} 2024-11-24T02:34:35,831 INFO [Time-limited test {}] server.Server(415): Started @18051ms 2024-11-24T02:34:35,831 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@66767106{HTTP/1.1, (http/1.1)}{localhost:37507} 2024-11-24T02:34:35,831 INFO [Thread-251 {}] server.Server(415): Started @18051ms 2024-11-24T02:34:35,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741837_1013 (size=5) 2024-11-24T02:34:36,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T02:34:36,601 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T02:34:36,610 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:34:36,610 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-24T02:34:36,611 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T02:34:36,611 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T02:34:36,611 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T02:34:36,611 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T02:34:36,867 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-24T02:34:36,872 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:36,913 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-24T02:34:36,913 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:34:36,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:34:36,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:34:36,919 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:34:36,920 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:36,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13757798{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:34:36,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38b35098{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-24T02:34:36,971 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-24T02:34:36,971 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-24T02:34:36,971 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-24T02:34:36,971 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-24T02:34:36,979 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T02:34:36,985 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T02:34:37,079 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T02:34:37,089 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7ac3fdb2{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/java.io.tmpdir/jetty-localhost-46261-hadoop-yarn-common-3_4_1_jar-_-any-8334354154094802490/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-24T02:34:37,090 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14fd9c4e{HTTP/1.1, (http/1.1)}{localhost:46261} 2024-11-24T02:34:37,090 INFO [Time-limited test {}] server.Server(415): Started @19310ms 2024-11-24T02:34:37,211 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-24T02:34:37,214 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:37,231 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-24T02:34:37,232 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:34:37,235 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:34:37,235 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:34:37,235 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:34:37,236 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:34:37,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ce56c02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:34:37,237 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17558f00{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-24T02:34:37,282 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-24T02:34:37,282 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-24T02:34:37,282 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-24T02:34:37,282 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-24T02:34:37,291 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T02:34:37,296 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T02:34:37,378 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-24T02:34:37,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a47ddee{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/java.io.tmpdir/jetty-localhost-39237-hadoop-yarn-common-3_4_1_jar-_-any-2259824249078679543/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-24T02:34:37,383 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2bf04c0f{HTTP/1.1, (http/1.1)}{localhost:39237} 2024-11-24T02:34:37,384 INFO [Time-limited test {}] server.Server(415): Started @19603ms 2024-11-24T02:34:37,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-24T02:34:37,407 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:40215/backupUT 2024-11-24T02:34:37,411 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:34:37,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:34:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77746083 2024-11-24T02:34:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:34:37,424 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:34:37,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-11-24T02:34:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-24T02:34:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:34:37,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 19 msec 2024-11-24T02:34:37,476 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T02:34:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:34:37,553 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-24T02:34:37,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-11-24T02:34:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-24T02:34:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-24T02:34:37,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 13 msec 2024-11-24T02:34:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-24T02:34:37,673 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-24T02:34:37,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-11-24T02:34:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-24T02:34:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-24T02:34:37,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 19 msec 2024-11-24T02:34:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-24T02:34:37,792 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-24T02:34:37,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-11-24T02:34:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-24T02:34:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:34:37,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 12 msec 2024-11-24T02:34:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:34:37,902 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-24T02:34:37,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1732415677407', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:34:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1732415677407 2024-11-24T02:34:37,922 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1732415677407 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:34:37,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1732415677407" procId is: 8 2024-11-24T02:34:37,924 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:37,926 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1732415677407 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:34:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T02:34:37,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741838_1014 (size=367) 2024-11-24T02:34:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T02:34:38,063 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-24T02:34:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T02:34:38,340 INFO [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a9fcfc65d96498f86ae4291b18ffd315, NAME => 'test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1732415677407', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:38,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741839_1015 (size=53) 2024-11-24T02:34:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T02:34:38,750 DEBUG [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(898): Instantiated test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:38,751 DEBUG [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(1722): Closing a9fcfc65d96498f86ae4291b18ffd315, disabling compactions & flushes 2024-11-24T02:34:38,751 INFO [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(1755): Closing region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:38,751 DEBUG [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:38,751 DEBUG [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. after waiting 0 ms 2024-11-24T02:34:38,751 DEBUG [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:38,751 INFO [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(1973): Closed test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:38,751 DEBUG [RegionOpenAndInit-test-1732415677407-pool-0 {}] regionserver.HRegion(1676): Region close journal for a9fcfc65d96498f86ae4291b18ffd315: Waiting for close lock at 1732415678751Disabling compacts and flushes for region at 1732415678751Disabling writes for close at 1732415678751Writing region close event to WAL at 1732415678751Closed at 1732415678751 2024-11-24T02:34:38,753 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1732415677407 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:34:38,759 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732415678753"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415678753"}]},"ts":"1732415678753"} 2024-11-24T02:34:38,763 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:34:38,765 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1732415677407 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:34:38,769 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732415677407","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415678766"}]},"ts":"1732415678766"} 2024-11-24T02:34:38,773 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732415677407, state=ENABLING in hbase:meta 2024-11-24T02:34:38,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, ASSIGN}] 2024-11-24T02:34:38,777 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, ASSIGN 2024-11-24T02:34:38,778 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:34:38,930 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:38,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, ASSIGN because future has completed 2024-11-24T02:34:38,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T02:34:39,108 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:39,108 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:34:39,108 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor a67e7d17e593,38807,1732415667927.regiongroup-1 2024-11-24T02:34:39,111 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1, suffix=, logDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927, archiveDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs, maxLogs=32 2024-11-24T02:34:39,127 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415679113, exclude list is [], retry=0 2024-11-24T02:34:39,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:34:39,134 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415679113 2024-11-24T02:34:39,134 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:34:39,135 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => a9fcfc65d96498f86ae4291b18ffd315, NAME => 'test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:39,135 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:34:39,135 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732415677407 a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,135 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:39,136 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,136 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,138 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,141 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9fcfc65d96498f86ae4291b18ffd315 columnFamilyName f 2024-11-24T02:34:39,141 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:39,143 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(327): Store=a9fcfc65d96498f86ae4291b18ffd315/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:39,143 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,145 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,145 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,146 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,146 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,149 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,153 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:34:39,153 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened a9fcfc65d96498f86ae4291b18ffd315; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10009108960, jitterRate=-0.06782908737659454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T02:34:39,154 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:39,155 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for a9fcfc65d96498f86ae4291b18ffd315: Running coprocessor pre-open hook at 1732415679136Writing region info on filesystem at 1732415679136Initializing all the Stores at 1732415679138 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415679138Cleaning up temporary data from old regions at 1732415679146 (+8 ms)Running coprocessor post-open hooks at 1732415679154 (+8 ms)Region opened successfully at 1732415679155 (+1 ms) 2024-11-24T02:34:39,157 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., pid=10, masterSystemTime=1732415679089 2024-11-24T02:34:39,162 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:39,162 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:39,162 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:39,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:34:39,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T02:34:39,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 in 235 msec 2024-11-24T02:34:39,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T02:34:39,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, ASSIGN in 401 msec 2024-11-24T02:34:39,182 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1732415677407 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:34:39,182 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732415677407","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415679182"}]},"ts":"1732415679182"} 2024-11-24T02:34:39,185 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732415677407, state=ENABLED in hbase:meta 2024-11-24T02:34:39,187 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1732415677407 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:34:39,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1732415677407 in 1.2740 sec 2024-11-24T02:34:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-24T02:34:40,072 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1732415677407 completed 2024-11-24T02:34:40,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1457674e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:40,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:34:40,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:34:40,075 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:34:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:34:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:34:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@682bfc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:34:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:34:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:40,078 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:34:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8983aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:40,083 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:34:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:34:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:34:40,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:34:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732415677407', row='row0', locateType=CURRENT is [region=test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:34:40,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(8528): writing data to region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. with WAL disabled. Data may be lost in the event of a crash. 2024-11-24T02:34:40,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17324156774071', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:34:40,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17324156774071 2024-11-24T02:34:40,263 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17324156774071 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:34:40,263 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:40,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17324156774071" procId is: 11 2024-11-24T02:34:40,265 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17324156774071 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:34:40,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:34:40,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741841_1017 (size=364) 2024-11-24T02:34:40,275 INFO [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 50af115aeed7c572d01a2d7d6548f037, NAME => 'ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17324156774071', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:40,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741842_1018 (size=50) 2024-11-24T02:34:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:34:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:34:40,684 DEBUG [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:40,684 DEBUG [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(1722): Closing 50af115aeed7c572d01a2d7d6548f037, disabling compactions & flushes 2024-11-24T02:34:40,684 INFO [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:40,684 DEBUG [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:40,684 DEBUG [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. after waiting 0 ms 2024-11-24T02:34:40,684 DEBUG [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:40,684 INFO [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:40,684 DEBUG [RegionOpenAndInit-ns2:test-17324156774071-pool-0 {}] regionserver.HRegion(1676): Region close journal for 50af115aeed7c572d01a2d7d6548f037: Waiting for close lock at 1732415680684Disabling compacts and flushes for region at 1732415680684Disabling writes for close at 1732415680684Writing region close event to WAL at 1732415680684Closed at 1732415680684 2024-11-24T02:34:40,686 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17324156774071 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:34:40,686 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732415680686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415680686"}]},"ts":"1732415680686"} 2024-11-24T02:34:40,690 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:34:40,692 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17324156774071 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:34:40,692 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17324156774071","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415680692"}]},"ts":"1732415680692"} 2024-11-24T02:34:40,695 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17324156774071, state=ENABLING in hbase:meta 2024-11-24T02:34:40,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17324156774071, region=50af115aeed7c572d01a2d7d6548f037, ASSIGN}] 2024-11-24T02:34:40,697 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17324156774071, region=50af115aeed7c572d01a2d7d6548f037, ASSIGN 2024-11-24T02:34:40,699 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17324156774071, region=50af115aeed7c572d01a2d7d6548f037, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:34:40,850 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=50af115aeed7c572d01a2d7d6548f037, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:40,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17324156774071, region=50af115aeed7c572d01a2d7d6548f037, ASSIGN because future has completed 2024-11-24T02:34:40,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50af115aeed7c572d01a2d7d6548f037, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:34:41,013 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:41,014 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 50af115aeed7c572d01a2d7d6548f037, NAME => 'ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:41,014 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:34:41,014 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17324156774071 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,014 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:41,014 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,015 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,017 INFO [StoreOpener-50af115aeed7c572d01a2d7d6548f037-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,020 INFO [StoreOpener-50af115aeed7c572d01a2d7d6548f037-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50af115aeed7c572d01a2d7d6548f037 columnFamilyName f 2024-11-24T02:34:41,020 DEBUG [StoreOpener-50af115aeed7c572d01a2d7d6548f037-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:41,021 INFO [StoreOpener-50af115aeed7c572d01a2d7d6548f037-1 {}] regionserver.HStore(327): Store=50af115aeed7c572d01a2d7d6548f037/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:41,021 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,022 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,023 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,024 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,024 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,027 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,030 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:34:41,031 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 50af115aeed7c572d01a2d7d6548f037; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10653673120, jitterRate=-0.00779937207698822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T02:34:41,031 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:41,032 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 50af115aeed7c572d01a2d7d6548f037: Running coprocessor pre-open hook at 1732415681015Writing region info on filesystem at 1732415681015Initializing all the Stores at 1732415681016 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415681016Cleaning up temporary data from old regions at 1732415681024 (+8 ms)Running coprocessor post-open hooks at 1732415681031 (+7 ms)Region opened successfully at 1732415681032 (+1 ms) 2024-11-24T02:34:41,034 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037., pid=13, masterSystemTime=1732415681007 2024-11-24T02:34:41,037 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:41,037 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:41,038 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=50af115aeed7c572d01a2d7d6548f037, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:41,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50af115aeed7c572d01a2d7d6548f037, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:34:41,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-24T02:34:41,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 50af115aeed7c572d01a2d7d6548f037, server=a67e7d17e593,38807,1732415667927 in 189 msec 2024-11-24T02:34:41,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T02:34:41,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17324156774071, region=50af115aeed7c572d01a2d7d6548f037, ASSIGN in 351 msec 2024-11-24T02:34:41,052 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17324156774071 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:34:41,052 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17324156774071","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415681052"}]},"ts":"1732415681052"} 2024-11-24T02:34:41,055 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17324156774071, state=ENABLED in hbase:meta 2024-11-24T02:34:41,057 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17324156774071 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:34:41,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17324156774071 in 798 msec 2024-11-24T02:34:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:34:41,402 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17324156774071 completed 2024-11-24T02:34:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17324156774071', row='row0', locateType=CURRENT is [region=ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:34:41,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(8528): writing data to region ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. with WAL disabled. Data may be lost in the event of a crash. 2024-11-24T02:34:41,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17324156774072', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:34:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17324156774072 2024-11-24T02:34:41,562 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17324156774072 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:34:41,562 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:41,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17324156774072" procId is: 14 2024-11-24T02:34:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T02:34:41,563 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17324156774072 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:34:41,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741843_1019 (size=390) 2024-11-24T02:34:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T02:34:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T02:34:41,974 INFO [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aca5da70eca26b7decb49ebc97e25da8, NAME => 'ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17324156774072', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741844_1020 (size=50) 2024-11-24T02:34:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T02:34:42,384 DEBUG [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:42,384 DEBUG [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(1722): Closing aca5da70eca26b7decb49ebc97e25da8, disabling compactions & flushes 2024-11-24T02:34:42,384 INFO [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:34:42,384 DEBUG [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:34:42,384 DEBUG [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. after waiting 0 ms 2024-11-24T02:34:42,384 DEBUG [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:34:42,384 INFO [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:34:42,384 DEBUG [RegionOpenAndInit-ns3:test-17324156774072-pool-0 {}] regionserver.HRegion(1676): Region close journal for aca5da70eca26b7decb49ebc97e25da8: Waiting for close lock at 1732415682384Disabling compacts and flushes for region at 1732415682384Disabling writes for close at 1732415682384Writing region close event to WAL at 1732415682384Closed at 1732415682384 2024-11-24T02:34:42,386 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17324156774072 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:34:42,387 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732415682387"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415682387"}]},"ts":"1732415682387"} 2024-11-24T02:34:42,390 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:34:42,392 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17324156774072 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:34:42,392 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17324156774072","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415682392"}]},"ts":"1732415682392"} 2024-11-24T02:34:42,394 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17324156774072, state=ENABLING in hbase:meta 2024-11-24T02:34:42,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17324156774072, region=aca5da70eca26b7decb49ebc97e25da8, ASSIGN}] 2024-11-24T02:34:42,397 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17324156774072, region=aca5da70eca26b7decb49ebc97e25da8, ASSIGN 2024-11-24T02:34:42,399 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17324156774072, region=aca5da70eca26b7decb49ebc97e25da8, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:34:42,549 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=aca5da70eca26b7decb49ebc97e25da8, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:42,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17324156774072, region=aca5da70eca26b7decb49ebc97e25da8, ASSIGN because future has completed 2024-11-24T02:34:42,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure aca5da70eca26b7decb49ebc97e25da8, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T02:34:42,712 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:34:42,713 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => aca5da70eca26b7decb49ebc97e25da8, NAME => 'ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:42,713 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:34:42,714 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17324156774072 aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,714 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:42,714 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,714 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,716 INFO [StoreOpener-aca5da70eca26b7decb49ebc97e25da8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,719 INFO [StoreOpener-aca5da70eca26b7decb49ebc97e25da8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aca5da70eca26b7decb49ebc97e25da8 columnFamilyName f 2024-11-24T02:34:42,719 DEBUG [StoreOpener-aca5da70eca26b7decb49ebc97e25da8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:42,720 INFO [StoreOpener-aca5da70eca26b7decb49ebc97e25da8-1 {}] regionserver.HStore(327): Store=aca5da70eca26b7decb49ebc97e25da8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:42,721 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,722 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns3/test-17324156774072/aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,722 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns3/test-17324156774072/aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,723 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,723 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,726 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,729 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns3/test-17324156774072/aca5da70eca26b7decb49ebc97e25da8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:34:42,730 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened aca5da70eca26b7decb49ebc97e25da8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11077340160, jitterRate=0.03165769577026367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T02:34:42,730 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:34:42,731 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for aca5da70eca26b7decb49ebc97e25da8: Running coprocessor pre-open hook at 1732415682714Writing region info on filesystem at 1732415682714Initializing all the Stores at 1732415682716 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415682716Cleaning up temporary data from old regions at 1732415682723 (+7 ms)Running coprocessor post-open hooks at 1732415682730 (+7 ms)Region opened successfully at 1732415682731 (+1 ms) 2024-11-24T02:34:42,733 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8., pid=16, masterSystemTime=1732415682707 2024-11-24T02:34:42,735 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:34:42,736 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:34:42,736 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=aca5da70eca26b7decb49ebc97e25da8, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:42,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure aca5da70eca26b7decb49ebc97e25da8, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:34:42,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T02:34:42,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure aca5da70eca26b7decb49ebc97e25da8, server=a67e7d17e593,38807,1732415667927 in 188 msec 2024-11-24T02:34:42,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T02:34:42,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17324156774072, region=aca5da70eca26b7decb49ebc97e25da8, ASSIGN in 350 msec 2024-11-24T02:34:42,749 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17324156774072 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:34:42,749 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17324156774072","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415682749"}]},"ts":"1732415682749"} 2024-11-24T02:34:42,752 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17324156774072, state=ENABLED in hbase:meta 2024-11-24T02:34:42,753 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17324156774072 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:34:42,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17324156774072 in 1.1950 sec 2024-11-24T02:34:43,478 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1732415677407' 2024-11-24T02:34:43,480 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17324156774071' 2024-11-24T02:34:43,481 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17324156774072' 2024-11-24T02:34:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-24T02:34:43,712 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17324156774072 completed 2024-11-24T02:34:43,712 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17324156774072 get assigned. Timeout = 60000ms 2024-11-24T02:34:43,713 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T02:34:43,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17324156774072 assigned to meta. Checking AM states. 2024-11-24T02:34:43,719 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T02:34:43,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17324156774072 assigned. 2024-11-24T02:34:43,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17324156774073', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:34:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17324156774073 2024-11-24T02:34:43,725 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17324156774073 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:34:43,725 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:43,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17324156774073" procId is: 17 2024-11-24T02:34:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:34:43,727 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17324156774073 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:34:43,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741845_1021 (size=390) 2024-11-24T02:34:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:34:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:34:44,168 INFO [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a605441c1cb1d296fb15e4b88f05f9c, NAME => 'ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17324156774073', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:44,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741846_1022 (size=50) 2024-11-24T02:34:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:34:44,578 DEBUG [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:44,578 DEBUG [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(1722): Closing 1a605441c1cb1d296fb15e4b88f05f9c, disabling compactions & flushes 2024-11-24T02:34:44,578 INFO [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:34:44,578 DEBUG [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:34:44,578 DEBUG [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. after waiting 0 ms 2024-11-24T02:34:44,578 DEBUG [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:34:44,578 INFO [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:34:44,578 DEBUG [RegionOpenAndInit-ns4:test-17324156774073-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a605441c1cb1d296fb15e4b88f05f9c: Waiting for close lock at 1732415684578Disabling compacts and flushes for region at 1732415684578Disabling writes for close at 1732415684578Writing region close event to WAL at 1732415684578Closed at 1732415684578 2024-11-24T02:34:44,580 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17324156774073 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:34:44,581 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732415684581"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415684581"}]},"ts":"1732415684581"} 2024-11-24T02:34:44,584 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:34:44,585 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17324156774073 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:34:44,585 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17324156774073","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415684585"}]},"ts":"1732415684585"} 2024-11-24T02:34:44,588 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17324156774073, state=ENABLING in hbase:meta 2024-11-24T02:34:44,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17324156774073, region=1a605441c1cb1d296fb15e4b88f05f9c, ASSIGN}] 2024-11-24T02:34:44,590 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17324156774073, region=1a605441c1cb1d296fb15e4b88f05f9c, ASSIGN 2024-11-24T02:34:44,591 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17324156774073, region=1a605441c1cb1d296fb15e4b88f05f9c, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:34:44,640 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-24T02:34:44,742 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=1a605441c1cb1d296fb15e4b88f05f9c, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:44,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17324156774073, region=1a605441c1cb1d296fb15e4b88f05f9c, ASSIGN because future has completed 2024-11-24T02:34:44,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a605441c1cb1d296fb15e4b88f05f9c, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:34:44,905 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:34:44,905 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a605441c1cb1d296fb15e4b88f05f9c, NAME => 'ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:44,905 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:34:44,906 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17324156774073 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,906 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:44,906 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,906 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,908 INFO [StoreOpener-1a605441c1cb1d296fb15e4b88f05f9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,911 INFO [StoreOpener-1a605441c1cb1d296fb15e4b88f05f9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a605441c1cb1d296fb15e4b88f05f9c columnFamilyName f 2024-11-24T02:34:44,911 DEBUG [StoreOpener-1a605441c1cb1d296fb15e4b88f05f9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:44,912 INFO [StoreOpener-1a605441c1cb1d296fb15e4b88f05f9c-1 {}] regionserver.HStore(327): Store=1a605441c1cb1d296fb15e4b88f05f9c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:44,912 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,913 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns4/test-17324156774073/1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,914 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns4/test-17324156774073/1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,915 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,915 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,918 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,921 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns4/test-17324156774073/1a605441c1cb1d296fb15e4b88f05f9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:34:44,922 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened 1a605441c1cb1d296fb15e4b88f05f9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10088222400, jitterRate=-0.060461074113845825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T02:34:44,922 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:34:44,923 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for 1a605441c1cb1d296fb15e4b88f05f9c: Running coprocessor pre-open hook at 1732415684906Writing region info on filesystem at 1732415684906Initializing all the Stores at 1732415684907 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415684907Cleaning up temporary data from old regions at 1732415684915 (+8 ms)Running coprocessor post-open hooks at 1732415684922 (+7 ms)Region opened successfully at 1732415684923 (+1 ms) 2024-11-24T02:34:44,925 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c., pid=19, masterSystemTime=1732415684900 2024-11-24T02:34:44,927 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:34:44,928 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:34:44,929 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=1a605441c1cb1d296fb15e4b88f05f9c, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:44,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a605441c1cb1d296fb15e4b88f05f9c, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:34:44,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-24T02:34:44,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a605441c1cb1d296fb15e4b88f05f9c, server=a67e7d17e593,38807,1732415667927 in 188 msec 2024-11-24T02:34:44,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T02:34:44,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17324156774073, region=1a605441c1cb1d296fb15e4b88f05f9c, ASSIGN in 349 msec 2024-11-24T02:34:44,942 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17324156774073 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:34:44,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17324156774073","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415684942"}]},"ts":"1732415684942"} 2024-11-24T02:34:44,945 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17324156774073, state=ENABLED in hbase:meta 2024-11-24T02:34:44,946 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17324156774073 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:34:44,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17324156774073 in 1.2240 sec 2024-11-24T02:34:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:34:45,872 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17324156774073 completed 2024-11-24T02:34:45,872 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17324156774073 get assigned. Timeout = 60000ms 2024-11-24T02:34:45,872 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T02:34:45,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17324156774073 assigned to meta. Checking AM states. 2024-11-24T02:34:45,877 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T02:34:45,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17324156774073 assigned. 2024-11-24T02:34:45,878 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:34:45,878 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:34:45,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:45,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:45,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:34:45,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=17, AvailableMemoryMB=5791 2024-11-24T02:34:45,906 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-11-24T02:34:45,907 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-24T02:34:45,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732415677407 from 'test-1732415677407', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732415677407', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:34:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732415677407 2024-11-24T02:34:45,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741847_1023 (size=908) 2024-11-24T02:34:46,337 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/.tabledesc/.tableinfo.0000000001.367 2024-11-24T02:34:46,337 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/.tabledesc/.tableinfo.0000000002.908 2024-11-24T02:34:46,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732415677407}] 2024-11-24T02:34:46,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE}] 2024-11-24T02:34:46,353 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE 2024-11-24T02:34:46,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=CLOSING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:46,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE because future has completed 2024-11-24T02:34:46,358 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:34:46,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:46,517 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:46,517 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:34:46,518 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing a9fcfc65d96498f86ae4291b18ffd315, disabling compactions & flushes 2024-11-24T02:34:46,518 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:46,518 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:46,518 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. after waiting 0 ms 2024-11-24T02:34:46,518 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:46,521 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing a9fcfc65d96498f86ae4291b18ffd315 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-24T02:34:46,582 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f/53a13e59c6ae40e384cc3ce799783392 is 37, key is row10/f:q1/1732415680115/Put/seqid=0 2024-11-24T02:34:46,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741848_1024 (size=8413) 2024-11-24T02:34:46,600 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17324156774071 2024-11-24T02:34:46,600 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17324156774071 Metrics about Tables on a single HBase RegionServer 2024-11-24T02:34:46,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17324156774072 2024-11-24T02:34:46,601 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17324156774072 Metrics about Tables on a single HBase RegionServer 2024-11-24T02:34:46,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732415677407 2024-11-24T02:34:46,602 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1732415677407 Metrics about Tables on a single HBase RegionServer 2024-11-24T02:34:46,603 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17324156774073 2024-11-24T02:34:46,603 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17324156774073 Metrics about Tables on a single HBase RegionServer 2024-11-24T02:34:46,992 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f/53a13e59c6ae40e384cc3ce799783392 2024-11-24T02:34:47,044 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f/53a13e59c6ae40e384cc3ce799783392 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 2024-11-24T02:34:47,054 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392, entries=99, sequenceid=103, filesize=8.2 K 2024-11-24T02:34:47,060 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for a9fcfc65d96498f86ae4291b18ffd315 in 539ms, sequenceid=103, compaction requested=false 2024-11-24T02:34:47,067 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-24T02:34:47,071 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:34:47,071 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:47,071 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for a9fcfc65d96498f86ae4291b18ffd315: Waiting for close lock at 1732415686518Running coprocessor pre-close hooks at 1732415686518Disabling compacts and flushes for region at 1732415686518Disabling writes for close at 1732415686518Obtaining lock to block concurrent updates at 1732415686521 (+3 ms)Preparing flush snapshotting stores in a9fcfc65d96498f86ae4291b18ffd315 at 1732415686522 (+1 ms)Finished memstore snapshotting test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1732415686528 (+6 ms)Flushing stores of test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. at 1732415686529 (+1 ms)Flushing a9fcfc65d96498f86ae4291b18ffd315/f: creating writer at 1732415686531 (+2 ms)Flushing a9fcfc65d96498f86ae4291b18ffd315/f: appending metadata at 1732415686571 (+40 ms)Flushing a9fcfc65d96498f86ae4291b18ffd315/f: closing flushed file at 1732415686573 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a9ddb0f: reopening flushed file at 1732415687043 (+470 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for a9fcfc65d96498f86ae4291b18ffd315 in 539ms, sequenceid=103, compaction requested=false at 1732415687060 (+17 ms)Writing region close event to WAL at 1732415687062 (+2 ms)Running coprocessor post-close hooks at 1732415687068 (+6 ms)Closed at 1732415687071 (+3 ms) 2024-11-24T02:34:47,071 WARN [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: a9fcfc65d96498f86ae4291b18ffd315 to self. 2024-11-24T02:34:47,074 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=CLOSED 2024-11-24T02:34:47,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:34:47,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-24T02:34:47,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 in 721 msec 2024-11-24T02:34:47,084 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE; state=CLOSED, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=true 2024-11-24T02:34:47,235 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:47,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE because future has completed 2024-11-24T02:34:47,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:47,397 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:47,398 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => a9fcfc65d96498f86ae4291b18ffd315, NAME => 'test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:47,398 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:34:47,398 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732415677407 a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,398 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:47,398 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,399 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,400 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,402 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9fcfc65d96498f86ae4291b18ffd315 columnFamilyName f 2024-11-24T02:34:47,402 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:47,413 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 2024-11-24T02:34:47,413 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(327): Store=a9fcfc65d96498f86ae4291b18ffd315/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:47,413 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,415 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9fcfc65d96498f86ae4291b18ffd315 columnFamilyName f3 2024-11-24T02:34:47,416 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:47,416 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(327): Store=a9fcfc65d96498f86ae4291b18ffd315/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:47,417 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,424 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9fcfc65d96498f86ae4291b18ffd315 columnFamilyName mob 2024-11-24T02:34:47,429 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:47,429 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(327): Store=a9fcfc65d96498f86ae4291b18ffd315/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:47,430 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,431 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,433 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,434 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,434 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,436 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732415677407 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T02:34:47,438 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,439 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened a9fcfc65d96498f86ae4291b18ffd315; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11258922240, jitterRate=0.04856884479522705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T02:34:47,439 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:47,440 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for a9fcfc65d96498f86ae4291b18ffd315: Running coprocessor pre-open hook at 1732415687399Writing region info on filesystem at 1732415687399Initializing all the Stores at 1732415687400 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415687400Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415687400Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415687400Cleaning up temporary data from old regions at 1732415687434 (+34 ms)Running coprocessor post-open hooks at 1732415687439 (+5 ms)Region opened successfully at 1732415687440 (+1 ms) 2024-11-24T02:34:47,442 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., pid=24, masterSystemTime=1732415687392 2024-11-24T02:34:47,444 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:47,444 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:47,445 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=OPEN, openSeqNum=107, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:47,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:34:47,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-24T02:34:47,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 in 212 msec 2024-11-24T02:34:47,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T02:34:47,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE in 1.1020 sec 2024-11-24T02:34:47,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-24T02:34:47,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732415677407 in 1.1170 sec 2024-11-24T02:34:47,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732415677407 in 1.5510 sec 2024-11-24T02:34:47,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-24T02:34:47,469 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732415677407 completed 2024-11-24T02:34:47,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99fab71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:47,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:34:47,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:34:47,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:34:47,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:34:47,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:34:47,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc133f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:47,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:34:47,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:34:47,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:47,474 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:34:47,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed7d35c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:47,476 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:34:47,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:34:47,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:34:47,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:34:47,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732415677407', row='row-test-1732415677407-3-0', locateType=CURRENT is [region=test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., hostname=a67e7d17e593,38807,1732415667927, seqNum=107] 2024-11-24T02:34:47,534 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:34:47,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:34:47,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e235c41 2024-11-24T02:34:47,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:34:47,538 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:34:47,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-11-24T02:34:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-24T02:34:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:34:47,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 12 msec 2024-11-24T02:34:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:34:47,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-24T02:34:47,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:34:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-24T02:34:47,681 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:34:47,681 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:47,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-24T02:34:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T02:34:47,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:34:47,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741849_1025 (size=635) 2024-11-24T02:34:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T02:34:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T02:34:48,095 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6df51c7a7a1591e81897afbbac0da0ea, NAME => 'backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:48,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741850_1026 (size=40) 2024-11-24T02:34:48,105 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:48,105 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 6df51c7a7a1591e81897afbbac0da0ea, disabling compactions & flushes 2024-11-24T02:34:48,105 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:48,105 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:48,105 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. after waiting 0 ms 2024-11-24T02:34:48,105 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:48,105 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:48,105 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6df51c7a7a1591e81897afbbac0da0ea: Waiting for close lock at 1732415688105Disabling compacts and flushes for region at 1732415688105Disabling writes for close at 1732415688105Writing region close event to WAL at 1732415688105Closed at 1732415688105 2024-11-24T02:34:48,107 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:34:48,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732415688107"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415688107"}]},"ts":"1732415688107"} 2024-11-24T02:34:48,111 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:34:48,112 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:34:48,113 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415688113"}]},"ts":"1732415688113"} 2024-11-24T02:34:48,116 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-24T02:34:48,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=6df51c7a7a1591e81897afbbac0da0ea, ASSIGN}] 2024-11-24T02:34:48,118 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=6df51c7a7a1591e81897afbbac0da0ea, ASSIGN 2024-11-24T02:34:48,120 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=6df51c7a7a1591e81897afbbac0da0ea, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:34:48,271 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=6df51c7a7a1591e81897afbbac0da0ea, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:48,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=6df51c7a7a1591e81897afbbac0da0ea, ASSIGN because future has completed 2024-11-24T02:34:48,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T02:34:48,432 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:48,432 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 6df51c7a7a1591e81897afbbac0da0ea, NAME => 'backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:48,433 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:34:48,433 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,433 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:48,433 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,433 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,435 INFO [StoreOpener-6df51c7a7a1591e81897afbbac0da0ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,438 INFO [StoreOpener-6df51c7a7a1591e81897afbbac0da0ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6df51c7a7a1591e81897afbbac0da0ea columnFamilyName meta 2024-11-24T02:34:48,438 DEBUG [StoreOpener-6df51c7a7a1591e81897afbbac0da0ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:48,439 INFO [StoreOpener-6df51c7a7a1591e81897afbbac0da0ea-1 {}] regionserver.HStore(327): Store=6df51c7a7a1591e81897afbbac0da0ea/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:48,439 INFO [StoreOpener-6df51c7a7a1591e81897afbbac0da0ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,441 INFO [StoreOpener-6df51c7a7a1591e81897afbbac0da0ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6df51c7a7a1591e81897afbbac0da0ea columnFamilyName session 2024-11-24T02:34:48,441 DEBUG [StoreOpener-6df51c7a7a1591e81897afbbac0da0ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:48,442 INFO [StoreOpener-6df51c7a7a1591e81897afbbac0da0ea-1 {}] regionserver.HStore(327): Store=6df51c7a7a1591e81897afbbac0da0ea/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:48,442 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,443 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,444 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,445 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,445 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,445 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T02:34:48,447 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,450 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:34:48,451 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 6df51c7a7a1591e81897afbbac0da0ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11779116480, jitterRate=0.09701570868492126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T02:34:48,451 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:48,452 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 6df51c7a7a1591e81897afbbac0da0ea: Running coprocessor pre-open hook at 1732415688433Writing region info on filesystem at 1732415688433Initializing all the Stores at 1732415688435 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415688435Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415688435Cleaning up temporary data from old regions at 1732415688445 (+10 ms)Running coprocessor post-open hooks at 1732415688451 (+6 ms)Region opened successfully at 1732415688452 (+1 ms) 2024-11-24T02:34:48,453 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., pid=28, masterSystemTime=1732415688427 2024-11-24T02:34:48,455 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:48,456 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:48,456 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=6df51c7a7a1591e81897afbbac0da0ea, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:48,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:34:48,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T02:34:48,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea, server=a67e7d17e593,38807,1732415667927 in 188 msec 2024-11-24T02:34:48,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-24T02:34:48,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=6df51c7a7a1591e81897afbbac0da0ea, ASSIGN in 349 msec 2024-11-24T02:34:48,470 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:34:48,470 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415688470"}]},"ts":"1732415688470"} 2024-11-24T02:34:48,473 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-24T02:34:48,474 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:34:48,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 796 msec 2024-11-24T02:34:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-24T02:34:48,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-24T02:34:48,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:34:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-24T02:34:48,842 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:34:48,842 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:48,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-24T02:34:48,844 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:34:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:34:48,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741851_1027 (size=640) 2024-11-24T02:34:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:34:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:34:49,254 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 48740e06c1a2c00320ec32fe075fe704, NAME => 'backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:49,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741852_1028 (size=45) 2024-11-24T02:34:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:34:49,484 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17324156774073' 2024-11-24T02:34:49,487 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-24T02:34:49,663 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:49,663 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing 48740e06c1a2c00320ec32fe075fe704, disabling compactions & flushes 2024-11-24T02:34:49,663 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:34:49,664 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:34:49,664 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. after waiting 0 ms 2024-11-24T02:34:49,664 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:34:49,664 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:34:49,664 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for 48740e06c1a2c00320ec32fe075fe704: Waiting for close lock at 1732415689663Disabling compacts and flushes for region at 1732415689663Disabling writes for close at 1732415689664 (+1 ms)Writing region close event to WAL at 1732415689664Closed at 1732415689664 2024-11-24T02:34:49,666 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:34:49,666 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732415689666"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415689666"}]},"ts":"1732415689666"} 2024-11-24T02:34:49,670 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:34:49,671 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:34:49,672 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415689671"}]},"ts":"1732415689671"} 2024-11-24T02:34:49,674 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-24T02:34:49,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=48740e06c1a2c00320ec32fe075fe704, ASSIGN}] 2024-11-24T02:34:49,676 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=48740e06c1a2c00320ec32fe075fe704, ASSIGN 2024-11-24T02:34:49,677 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=48740e06c1a2c00320ec32fe075fe704, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:34:49,828 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=48740e06c1a2c00320ec32fe075fe704, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:49,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=48740e06c1a2c00320ec32fe075fe704, ASSIGN because future has completed 2024-11-24T02:34:49,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48740e06c1a2c00320ec32fe075fe704, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:34:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:34:49,997 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:34:49,997 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 48740e06c1a2c00320ec32fe075fe704, NAME => 'backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:34:49,998 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:34:49,998 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:49,998 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:34:49,998 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:49,998 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,002 INFO [StoreOpener-48740e06c1a2c00320ec32fe075fe704-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,004 INFO [StoreOpener-48740e06c1a2c00320ec32fe075fe704-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48740e06c1a2c00320ec32fe075fe704 columnFamilyName meta 2024-11-24T02:34:50,004 DEBUG [StoreOpener-48740e06c1a2c00320ec32fe075fe704-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:50,004 INFO [StoreOpener-48740e06c1a2c00320ec32fe075fe704-1 {}] regionserver.HStore(327): Store=48740e06c1a2c00320ec32fe075fe704/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:50,005 INFO [StoreOpener-48740e06c1a2c00320ec32fe075fe704-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,006 INFO [StoreOpener-48740e06c1a2c00320ec32fe075fe704-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48740e06c1a2c00320ec32fe075fe704 columnFamilyName session 2024-11-24T02:34:50,006 DEBUG [StoreOpener-48740e06c1a2c00320ec32fe075fe704-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:50,007 INFO [StoreOpener-48740e06c1a2c00320ec32fe075fe704-1 {}] regionserver.HStore(327): Store=48740e06c1a2c00320ec32fe075fe704/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:34:50,007 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,008 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system_bulk/48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,009 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system_bulk/48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,010 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,010 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,011 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T02:34:50,012 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,015 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system_bulk/48740e06c1a2c00320ec32fe075fe704/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:34:50,016 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 48740e06c1a2c00320ec32fe075fe704; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9839700320, jitterRate=-0.08360649645328522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T02:34:50,016 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:34:50,016 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 48740e06c1a2c00320ec32fe075fe704: Running coprocessor pre-open hook at 1732415689999Writing region info on filesystem at 1732415689999Initializing all the Stores at 1732415690001 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415690001Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415690002 (+1 ms)Cleaning up temporary data from old regions at 1732415690010 (+8 ms)Running coprocessor post-open hooks at 1732415690016 (+6 ms)Region opened successfully at 1732415690016 2024-11-24T02:34:50,017 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704., pid=31, masterSystemTime=1732415689990 2024-11-24T02:34:50,020 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:34:50,020 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:34:50,021 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=48740e06c1a2c00320ec32fe075fe704, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:34:50,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48740e06c1a2c00320ec32fe075fe704, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:34:50,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T02:34:50,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 48740e06c1a2c00320ec32fe075fe704, server=a67e7d17e593,38807,1732415667927 in 188 msec 2024-11-24T02:34:50,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-24T02:34:50,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=48740e06c1a2c00320ec32fe075fe704, ASSIGN in 353 msec 2024-11-24T02:34:50,030 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:34:50,031 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415690030"}]},"ts":"1732415690030"} 2024-11-24T02:34:50,033 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-24T02:34:50,033 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:34:50,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.1950 sec 2024-11-24T02:34:50,411 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-24T02:34:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:34:50,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-24T02:34:51,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:34:51,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:34:51,018 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T02:34:51,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:34:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-24T02:34:51,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T02:34:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415691042 (current time:1732415691042). 2024-11-24T02:34:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:34:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-24T02:34:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@789a5df4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:34:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:34:51,045 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:34:51,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:34:51,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:34:51,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20522329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:51,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:34:51,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:34:51,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:51,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:34:51,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66afe9ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:34:51,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:34:51,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:34:51,053 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:34:51,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:34:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:34:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:51,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:34:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:34:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T02:34:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-24T02:34:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T02:34:51,065 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:34:51,070 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:34:51,082 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:34:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741853_1029 (size=52) 2024-11-24T02:34:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T02:34:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T02:34:51,493 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:34:51,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea}] 2024-11-24T02:34:51,504 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:51,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-24T02:34:51,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:51,663 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 6df51c7a7a1591e81897afbbac0da0ea 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-24T02:34:51,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/dc953a38304d4f09b0a4aa66f11b529d is 49, key is activesession:/session:c/1732415691031/Put/seqid=0 2024-11-24T02:34:51,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741854_1030 (size=5049) 2024-11-24T02:34:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T02:34:52,089 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/dc953a38304d4f09b0a4aa66f11b529d 2024-11-24T02:34:52,107 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/dc953a38304d4f09b0a4aa66f11b529d as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d 2024-11-24T02:34:52,115 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d, entries=1, sequenceid=5, filesize=4.9 K 2024-11-24T02:34:52,117 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 6df51c7a7a1591e81897afbbac0da0ea in 454ms, sequenceid=5, compaction requested=false 2024-11-24T02:34:52,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 6df51c7a7a1591e81897afbbac0da0ea: 2024-11-24T02:34:52,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. for snapshot_backup_system completed. 2024-11-24T02:34:52,118 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.' region-info for snapshot=snapshot_backup_system 2024-11-24T02:34:52,121 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:34:52,123 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T02:34:52,123 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d] hfiles 2024-11-24T02:34:52,125 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d for snapshot=snapshot_backup_system 2024-11-24T02:34:52,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741855_1031 (size=95) 2024-11-24T02:34:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T02:34:52,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:34:52,541 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-24T02:34:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-24T02:34:52,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:52,544 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:34:52,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-24T02:34:52,558 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:34:52,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea in 1.0590 sec 2024-11-24T02:34:52,560 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:34:52,561 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:34:52,561 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-24T02:34:52,564 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-24T02:34:52,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741856_1032 (size=731) 2024-11-24T02:34:52,990 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:34:53,003 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:34:53,004 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_backup_system 2024-11-24T02:34:53,006 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:34:53,006 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-24T02:34:53,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9460 sec 2024-11-24T02:34:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T02:34:53,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-24T02:34:53,214 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732415687521 started at 1732415693213. 2024-11-24T02:34:53,233 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732415687521 has been started. 2024-11-24T02:34:53,237 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-24T02:34:53,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-24T02:34:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-24T02:34:53,245 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-24T02:34:53,245 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-24T02:34:53,245 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-24T02:34:53,246 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-24T02:34:53,246 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,246 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T02:34:53,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T02:34:53,308 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,309 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T02:34:53,309 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:34:53,310 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,310 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-24T02:34:53,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-24T02:34:53,312 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-24T02:34:53,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T02:34:53,314 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-24T02:34:53,316 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-24T02:34:53,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-24T02:34:53,317 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-24T02:34:53,317 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-24T02:34:53,317 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-24T02:34:53,317 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-24T02:34:53,317 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'a67e7d17e593,38807,1732415667927' joining acquired barrier for procedure (rolllog) in zk 2024-11-24T02:34:53,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,329 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-24T02:34:53,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:34:53,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:34:53,330 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-24T02:34:53,330 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-24T02:34:53,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:34:53,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:34:53,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:34:53,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:34:53,334 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'a67e7d17e593,38807,1732415667927' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-24T02:34:53,334 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@29679bbe[Count = 0] remaining members to acquire global barrier 2024-11-24T02:34:53,334 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-24T02:34:53,334 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-24T02:34:53,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:34:53,346 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-24T02:34:53,346 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-24T02:34:53,346 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-24T02:34:53,347 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,347 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-24T02:34:53,348 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-24T02:34:53,348 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,358 INFO [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732415671672 highest: 1732415679113 on a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,358 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0:(num 1732415671672) roll requested 2024-11-24T02:34:53,370 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359, exclude list is [], retry=0 2024-11-24T02:34:53,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:34:53,376 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415671672 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 2024-11-24T02:34:53,376 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:34:53,376 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415671672 is not closed yet, will try archiving it next time 2024-11-24T02:34:53,376 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.meta:.meta(num 1732415672505) roll requested 2024-11-24T02:34:53,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741833_1009 (size=1034) 2024-11-24T02:34:53,380 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415671672 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415671672 2024-11-24T02:34:53,390 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta, exclude list is [], retry=0 2024-11-24T02:34:53,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:34:53,395 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415672505.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta 2024-11-24T02:34:53,395 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:34:53,395 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415672505.meta is not closed yet, will try archiving it next time 2024-11-24T02:34:53,395 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1:(num 1732415679113) roll requested 2024-11-24T02:34:53,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741834_1010 (size=12055) 2024-11-24T02:34:53,397 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415672505.meta not finished, retry = 0 2024-11-24T02:34:53,407 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395, exclude list is [], retry=0 2024-11-24T02:34:53,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:34:53,412 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415679113 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 2024-11-24T02:34:53,412 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:34:53,412 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415679113 is not closed yet, will try archiving it next time 2024-11-24T02:34:53,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741840_1016 (size=4756) 2024-11-24T02:34:53,414 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415679113 not finished, retry = 0 2024-11-24T02:34:53,419 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-24T02:34:53,419 INFO [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732415693359 on a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,419 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:34:53,420 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:34:53,420 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14954e1f 2024-11-24T02:34:53,420 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:34:53,421 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34351, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T02:34:53,422 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:34:53,422 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:34:53,422 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:34:53,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59055, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T02:34:53,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:34:53,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:34:53,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:40215/backupUT\x00', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:34:53,453 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-24T02:34:53,453 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'a67e7d17e593,38807,1732415667927' in zk 2024-11-24T02:34:53,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,466 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-24T02:34:53,466 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T02:34:53,466 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,466 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:34:53,466 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:34:53,466 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-24T02:34:53,467 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:34:53,467 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:34:53,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:34:53,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:34:53,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:34:53,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'a67e7d17e593,38807,1732415667927': 2024-11-24T02:34:53,470 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-24T02:34:53,470 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-24T02:34:53,471 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-24T02:34:53,471 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'a67e7d17e593,38807,1732415667927' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-24T02:34:53,471 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-24T02:34:53,471 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-24T02:34:53,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T02:34:53,483 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:34:53,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:34:53,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:34:53,483 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T02:34:53,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:34:53,484 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:34:53,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:34:53,484 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:34:53,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,485 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:34:53,485 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:34:53,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:34:53,499 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:34:53,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:34:53,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:34:53,500 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T02:34:53,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-24T02:34:53,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-24T02:34:53,500 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T02:34:53,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:34:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-24T02:34:53,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } 2024-11-24T02:34:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415693518 (current time:1732415693518). 2024-11-24T02:34:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:34:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732415693517_default_test-1732415677407 VERSION not specified, setting to 2 2024-11-24T02:34:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6ffbf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:34:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:34:53,520 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:34:53,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:34:53,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:34:53,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f7d8611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:53,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:34:53,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:34:53,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:53,522 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:34:53,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f59611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:34:53,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:34:53,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:34:53,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:34:53,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:34:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:34:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:53,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:34:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:34:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } 2024-11-24T02:34:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-24T02:34:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T02:34:53,531 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:34:53,533 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:34:53,536 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:34:53,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741860_1036 (size=84) 2024-11-24T02:34:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T02:34:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T02:34:53,950 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:34:53,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a9fcfc65d96498f86ae4291b18ffd315}] 2024-11-24T02:34:53,953 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:54,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-24T02:34:54,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:54,109 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing a9fcfc65d96498f86ae4291b18ffd315 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-24T02:34:54,132 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f3/dc0694660d6d4041923141ad81e67545 is 58, key is row-test-1732415677407-3-0/f3:q1/1732415687484/Put/seqid=0 2024-11-24T02:34:54,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741861_1037 (size=5387) 2024-11-24T02:34:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T02:34:54,537 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f3/dc0694660d6d4041923141ad81e67545 2024-11-24T02:34:54,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741862_1038 (size=4469) 2024-11-24T02:34:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T02:34:54,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/mob/d188d82f983547d6962f28b22a0c0e2c is 59, key is row-test-1732415677407-3-0/mob:q1/1732415687503/Put/seqid=0 2024-11-24T02:34:54,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741863_1039 (size=5484) 2024-11-24T02:34:55,160 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:34:55,392 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/mob/d188d82f983547d6962f28b22a0c0e2c 2024-11-24T02:34:55,404 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f3/dc0694660d6d4041923141ad81e67545 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 2024-11-24T02:34:55,413 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545, entries=6, sequenceid=121, filesize=5.3 K 2024-11-24T02:34:55,414 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/mob/d188d82f983547d6962f28b22a0c0e2c as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c 2024-11-24T02:34:55,422 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c, entries=6, sequenceid=121, filesize=5.4 K 2024-11-24T02:34:55,424 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for a9fcfc65d96498f86ae4291b18ffd315 in 1314ms, sequenceid=121, compaction requested=false 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for a9fcfc65d96498f86ae4291b18ffd315: 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. for snapshot_1732415693517_default_test-1732415677407 completed. 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.' region-info for snapshot=snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392] hfiles 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 for snapshot=snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545] hfiles 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 for snapshot=snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c] hfiles 2024-11-24T02:34:55,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c for snapshot=snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:55,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741864_1040 (size=185) 2024-11-24T02:34:55,501 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-24T02:34:55,534 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-24T02:34:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T02:34:55,831 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:34:55,831 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-24T02:34:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-24T02:34:55,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732415693517_default_test-1732415677407 on region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:55,832 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:34:55,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-24T02:34:55,836 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:34:55,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a9fcfc65d96498f86ae4291b18ffd315 in 1.8830 sec 2024-11-24T02:34:55,837 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:34:55,839 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-24T02:34:55,839 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-24T02:34:55,840 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:55,840 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-24T02:34:55,840 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:55,841 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-24T02:34:55,841 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:34:55,842 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-24T02:34:55,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741865_1041 (size=49) 2024-11-24T02:34:56,252 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:34:56,252 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:56,254 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:56,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741866_1042 (size=1146) 2024-11-24T02:34:56,600 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-24T02:34:56,600 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-24T02:34:56,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-24T02:34:56,602 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-24T02:34:56,672 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:34:56,681 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:34:56,682 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415693517_default_test-1732415677407 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:56,684 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:34:56,684 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-24T02:34:56,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732415693517_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } in 3.1560 sec 2024-11-24T02:34:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T02:34:57,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732415677407 completed 2024-11-24T02:34:57,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } 2024-11-24T02:34:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415697713 (current time:1732415697713). 2024-11-24T02:34:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:34:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732415697712_ns2_test-17324156774071 VERSION not specified, setting to 2 2024-11-24T02:34:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a82a975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:34:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:34:57,716 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:34:57,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:34:57,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:34:57,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591de230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:57,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:34:57,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:34:57,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:57,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:34:57,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371a9c08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:34:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:34:57,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:34:57,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:34:57,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:34:57,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:34:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:34:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:34:57,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:34:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:34:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } 2024-11-24T02:34:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-24T02:34:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T02:34:57,729 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:34:57,730 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:34:57,734 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:34:57,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741867_1043 (size=86) 2024-11-24T02:34:57,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T02:34:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T02:34:58,143 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:34:58,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 50af115aeed7c572d01a2d7d6548f037}] 2024-11-24T02:34:58,144 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:58,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-24T02:34:58,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:58,300 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 50af115aeed7c572d01a2d7d6548f037 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-24T02:34:58,323 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/.tmp/f/4a72dce212c94e28ad913a969e06b255 is 37, key is row10/f:q1/1732415681430/Put/seqid=0 2024-11-24T02:34:58,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741868_1044 (size=8413) 2024-11-24T02:34:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T02:34:58,729 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/.tmp/f/4a72dce212c94e28ad913a969e06b255 2024-11-24T02:34:58,741 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/.tmp/f/4a72dce212c94e28ad913a969e06b255 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 2024-11-24T02:34:58,749 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255, entries=99, sequenceid=103, filesize=8.2 K 2024-11-24T02:34:58,751 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 50af115aeed7c572d01a2d7d6548f037 in 451ms, sequenceid=103, compaction requested=false 2024-11-24T02:34:58,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 50af115aeed7c572d01a2d7d6548f037: 2024-11-24T02:34:58,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. for snapshot_1732415697712_ns2_test-17324156774071 completed. 2024-11-24T02:34:58,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037.' region-info for snapshot=snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:34:58,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:34:58,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255] hfiles 2024-11-24T02:34:58,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 for snapshot=snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:34:58,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741869_1045 (size=91) 2024-11-24T02:34:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T02:34:59,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:34:59,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-24T02:34:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-24T02:34:59,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732415697712_ns2_test-17324156774071 on region 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:59,159 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:34:59,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-24T02:34:59,162 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:34:59,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 50af115aeed7c572d01a2d7d6548f037 in 1.0170 sec 2024-11-24T02:34:59,163 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:34:59,164 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:34:59,164 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:34:59,165 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:34:59,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741870_1046 (size=456) 2024-11-24T02:34:59,580 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:34:59,588 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:34:59,588 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415697712_ns2_test-17324156774071 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:34:59,590 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:34:59,590 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-24T02:34:59,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732415697712_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } in 1.8640 sec 2024-11-24T02:34:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-24T02:34:59,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17324156774071 completed 2024-11-24T02:34:59,874 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732415687521 2024-11-24T02:34:59,874 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-24T02:34:59,876 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-24T02:34:59,877 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732415687521_test-1732415677407 2024-11-24T02:34:59,878 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732415693517_default_test-1732415677407 to hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/ 2024-11-24T02:34:59,883 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T02:34:59,913 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:40215, inputRoot=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:34:59,913 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_10852338_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/.hbase-snapshot/.tmp/snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:59,915 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T02:34:59,922 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415693517_default_test-1732415677407 to hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/.hbase-snapshot/.tmp/snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:34:59,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741871_1047 (size=84) 2024-11-24T02:34:59,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741872_1048 (size=1146) 2024-11-24T02:35:00,340 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:00,340 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:00,341 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,240 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-2745847428723224121.jar 2024-11-24T02:35:01,241 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,241 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,241 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,241 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,242 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,242 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,242 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,242 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,243 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:01,243 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:01,243 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:35:01,243 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:35:01,244 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:35:01,244 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:35:01,244 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:35:01,244 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:35:01,245 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:35:01,245 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:35:01,245 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:35:01,245 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:35:01,247 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:01,247 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:01,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:01,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:01,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:01,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:01,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:01,306 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:01,338 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732415693517_default_test-1732415677407' hfile list 2024-11-24T02:35:01,344 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-11-24T02:35:01,545 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1141310023_0001_08b1b920-a82a-46d2-bb7b-c45e15f9dc68/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:35:01,545 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1141310023_0001_08b1b920-a82a-46d2-bb7b-c45e15f9dc68/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:35:01,666 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-24T02:35:01,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741873_1049 (size=8413) 2024-11-24T02:35:02,143 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 output=hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 2024-11-24T02:35:02,144 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.017M/sec 2024-11-24T02:35:02,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741874_1050 (size=5484) 2024-11-24T02:35:02,583 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c output=hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c 2024-11-24T02:35:02,583 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-24T02:35:02,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741875_1051 (size=5387) 2024-11-24T02:35:03,053 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 output=hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 2024-11-24T02:35:03,053 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-11-24T02:35:03,647 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T02:35:03,648 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T02:35:03,657 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:35:03,657 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732415693517_default_test-1732415677407 finished. 2024-11-24T02:35:03,657 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732415687521_ns2:test-17324156774071 2024-11-24T02:35:03,657 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732415697712_ns2_test-17324156774071 to hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/ 2024-11-24T02:35:03,657 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T02:35:03,687 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:40215, inputRoot=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:35:03,687 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_10852338_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/.hbase-snapshot/.tmp/snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:35:03,689 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T02:35:03,695 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415697712_ns2_test-17324156774071 to hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/.hbase-snapshot/.tmp/snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:35:03,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741877_1053 (size=456) 2024-11-24T02:35:03,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741876_1052 (size=86) 2024-11-24T02:35:04,113 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,114 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-7005218489925490679.jar 2024-11-24T02:35:04,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,987 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,988 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,989 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,989 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,989 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:04,989 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:04,990 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:35:04,990 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:35:04,990 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:35:04,990 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:35:04,991 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:35:04,991 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:35:04,991 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:35:04,991 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:35:04,992 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:35:04,992 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:35:04,992 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:04,992 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:04,993 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:04,993 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:04,993 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:04,993 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:04,993 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:05,031 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:05,054 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732415697712_ns2_test-17324156774071' hfile list 2024-11-24T02:35:05,057 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-11-24T02:35:05,142 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1126146745_0002_20f98209-f216-4b2f-86df-4d70008c1b1b/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:35:05,142 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1126146745_0002_20f98209-f216-4b2f-86df-4d70008c1b1b/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:35:05,219 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-24T02:35:05,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741878_1054 (size=8413) 2024-11-24T02:35:05,662 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 output=hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 2024-11-24T02:35:05,662 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-24T02:35:06,189 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T02:35:06,190 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T02:35:06,199 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:35:06,199 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732415697712_ns2_test-17324156774071 finished. 2024-11-24T02:35:06,200 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732415677407 2024-11-24T02:35:06,200 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17324156774071 2024-11-24T02:35:06,243 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732415687521 2024-11-24T02:35:06,243 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-24T02:35:06,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741879_1055 (size=130) 2024-11-24T02:35:06,654 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:40215/backupUT/backup_1732415687521/.backup.manifest 2024-11-24T02:35:06,660 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732415687521 finished: type=FULL,tablelist=test-1732415677407;ns2:test-17324156774071,targetRootDir=hdfs://localhost:40215/backupUT,startts=1732415693213,completets=1732415706241,bytescopied=0 2024-11-24T02:35:06,660 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-24T02:35:06,661 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:35:06,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732415693517_default_test-1732415677407" type: DISABLED 2024-11-24T02:35:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732415693517_default_test-1732415677407 2024-11-24T02:35:06,671 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732415693517_default_test-1732415677407 for backup backup_1732415687521 succeeded. 2024-11-24T02:35:06,671 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:35:06,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732415697712_ns2_test-17324156774071" type: DISABLED 2024-11-24T02:35:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732415697712_ns2_test-17324156774071 2024-11-24T02:35:06,676 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732415697712_ns2_test-17324156774071 for backup backup_1732415687521 succeeded. 2024-11-24T02:35:06,677 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T02:35:06,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-24T02:35:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-24T02:35:06,690 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-24T02:35:06,696 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T02:35:06,711 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732415687521 completed. 2024-11-24T02:35:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1732415687521', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:35:06,906 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1732415677407 2024-11-24T02:35:06,923 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1732415677407 2024-11-24T02:35:07,100 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1732415677407 to Mob enabled CF 2024-11-24T02:35:07,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17324156774071', row='row-t20', locateType=CURRENT is [region=ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:35:07,141 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17324156774071 2024-11-24T02:35:07,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:07,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] assignment.SplitTableRegionProcedure(210): test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. splitKey isn't explicitly specified, will try to find a best split key from RS a67e7d17e593,38807,1732415667927 2024-11-24T02:35:07,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 because midkey is the same as first or last row 2024-11-24T02:35:07,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 because midkey is the same as first or last row 2024-11-24T02:35:07,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c because midkey is the same as first or last row 2024-11-24T02:35:07,154 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegion(2902): Flushing a9fcfc65d96498f86ae4291b18ffd315 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-24T02:35:07,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f/30c452fa45f2462f8bdd8c1b3f2c0f4e is 59, key is row-test-1732415677407-1-10/f:q1/1732415706753/Put/seqid=0 2024-11-24T02:35:07,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741880_1056 (size=10796) 2024-11-24T02:35:07,580 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f/30c452fa45f2462f8bdd8c1b3f2c0f4e 2024-11-24T02:35:07,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315 is 63, key is row-test-1732415677407-3-100/mob:q1/1732415707081/Put/seqid=0 2024-11-24T02:35:07,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741881_1057 (size=5591) 2024-11-24T02:35:08,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:35:08,014 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79/mob/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:08,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/mob/5d3f9bfaddac4bf4a79e8d16491044f6, store: [table=test-1732415677407 family=mob region=a9fcfc65d96498f86ae4291b18ffd315] 2024-11-24T02:35:08,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/mob/5d3f9bfaddac4bf4a79e8d16491044f6 is 192, key is row-test-1732415677407-3-100/mob:q1/1732415707081/Put/seqid=0 2024-11-24T02:35:08,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741882_1058 (size=13760) 2024-11-24T02:35:08,423 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/mob/5d3f9bfaddac4bf4a79e8d16491044f6 2024-11-24T02:35:08,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f/30c452fa45f2462f8bdd8c1b3f2c0f4e as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e 2024-11-24T02:35:08,445 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e, entries=99, sequenceid=334, filesize=10.5 K 2024-11-24T02:35:08,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/mob/5d3f9bfaddac4bf4a79e8d16491044f6 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6 2024-11-24T02:35:08,453 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6, entries=111, sequenceid=334, filesize=13.4 K 2024-11-24T02:35:08,454 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for a9fcfc65d96498f86ae4291b18ffd315 in 1300ms, sequenceid=334, compaction requested=false 2024-11-24T02:35:08,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.HRegion(2603): Flush status journal for a9fcfc65d96498f86ae4291b18ffd315: 2024-11-24T02:35:08,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e because midkey is the same as first or last row 2024-11-24T02:35:08,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 because midkey is the same as first or last row 2024-11-24T02:35:08,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6 because midkey is the same as first or last row 2024-11-24T02:35:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=a67e7d17e593,38807,1732415667927 2024-11-24T02:35:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:35:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.CallRunner(138): callId: 302 service: MasterService methodName: SplitRegion size: 98 connection: 172.17.0.2:33302 deadline: 1732415767148, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-24T02:35:08,461 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:08,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1732415677407 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:08,462 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T02:35:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732415677407,, stopping at row=test-1732415677407 ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,472 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732415677407,, stopping at row=test-1732415677407 ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,479 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1338 2024-11-24T02:35:08,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,521 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T02:35:08,525 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732415708479 2024-11-24T02:35:08,531 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732415687521} 2024-11-24T02:35:08,531 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-24T02:35:08,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:08,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T02:35:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415708579 (current time:1732415708579). 2024-11-24T02:35:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:35:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-24T02:35:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23338e81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:08,581 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:08,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:08,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:08,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eafd97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:08,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:08,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:08,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:08,583 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:08,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c694845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:08,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:08,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:08,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:08,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:35:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:08,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:35:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T02:35:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-24T02:35:08,590 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:35:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T02:35:08,591 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:35:08,594 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:35:08,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741883_1059 (size=52) 2024-11-24T02:35:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T02:35:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T02:35:09,003 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:35:09,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea}] 2024-11-24T02:35:09,006 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:09,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-24T02:35:09,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:35:09,160 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 6df51c7a7a1591e81897afbbac0da0ea 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-24T02:35:09,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/09439360a3384a58b0af1fd4ddfc50f5 is 158, key is trslm:hdfs://localhost:40215/backupUT\x00ns2:test-17324156774071/meta:log-roll-map/1732415706230/Put/seqid=0 2024-11-24T02:35:09,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741884_1060 (size=5914) 2024-11-24T02:35:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T02:35:09,587 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/09439360a3384a58b0af1fd4ddfc50f5 2024-11-24T02:35:09,604 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/bc9a95a157c24f92aeeba767a22b2c63 is 433, key is session:backup_1732415687521/session:context/1732415706695/Put/seqid=0 2024-11-24T02:35:09,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741885_1061 (size=5535) 2024-11-24T02:35:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T02:35:10,010 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/bc9a95a157c24f92aeeba767a22b2c63 2024-11-24T02:35:10,024 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/09439360a3384a58b0af1fd4ddfc50f5 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/09439360a3384a58b0af1fd4ddfc50f5 2024-11-24T02:35:10,032 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/09439360a3384a58b0af1fd4ddfc50f5, entries=6, sequenceid=17, filesize=5.8 K 2024-11-24T02:35:10,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/bc9a95a157c24f92aeeba767a22b2c63 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/bc9a95a157c24f92aeeba767a22b2c63 2024-11-24T02:35:10,040 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/bc9a95a157c24f92aeeba767a22b2c63, entries=2, sequenceid=17, filesize=5.4 K 2024-11-24T02:35:10,041 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 6df51c7a7a1591e81897afbbac0da0ea in 882ms, sequenceid=17, compaction requested=false 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 6df51c7a7a1591e81897afbbac0da0ea: 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. for snapshot_backup_system completed. 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.' region-info for snapshot=snapshot_backup_system 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/09439360a3384a58b0af1fd4ddfc50f5] hfiles 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/09439360a3384a58b0af1fd4ddfc50f5 for snapshot=snapshot_backup_system 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/bc9a95a157c24f92aeeba767a22b2c63] hfiles 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d for snapshot=snapshot_backup_system 2024-11-24T02:35:10,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/bc9a95a157c24f92aeeba767a22b2c63 for snapshot=snapshot_backup_system 2024-11-24T02:35:10,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741886_1062 (size=173) 2024-11-24T02:35:10,448 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:35:10,449 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-24T02:35:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-24T02:35:10,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:10,451 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:10,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-24T02:35:10,457 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:35:10,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea in 1.4500 sec 2024-11-24T02:35:10,459 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:35:10,459 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:35:10,459 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-24T02:35:10,460 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-24T02:35:10,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741887_1063 (size=810) 2024-11-24T02:35:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T02:35:10,877 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:35:10,888 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:35:10,888 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_backup_system 2024-11-24T02:35:10,890 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:35:10,890 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-24T02:35:10,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3020 sec 2024-11-24T02:35:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-24T02:35:12,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-24T02:35:12,753 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732415708479 started at 1732415712753. 2024-11-24T02:35:12,758 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732415708479 has been started. 2024-11-24T02:35:12,766 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [test-1732415677407, ns2:test-17324156774071] 2024-11-24T02:35:12,770 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732415679113for backupID backup_1732415708479 2024-11-24T02:35:12,770 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-24T02:35:12,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-24T02:35:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-24T02:35:12,771 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-24T02:35:12,772 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-24T02:35:12,772 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-24T02:35:12,772 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-24T02:35:12,772 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:12,772 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:12,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T02:35:12,845 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,846 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T02:35:12,846 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:35:12,847 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,847 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-24T02:35:12,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:12,850 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:12,851 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-24T02:35:12,851 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:12,851 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-24T02:35:12,851 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-24T02:35:12,851 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-24T02:35:12,851 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-24T02:35:12,852 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-24T02:35:12,852 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-24T02:35:12,852 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-24T02:35:12,852 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'a67e7d17e593,38807,1732415667927' joining acquired barrier for procedure (rolllog) in zk 2024-11-24T02:35:12,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,861 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:12,862 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,862 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:35:12,862 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:35:12,862 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-24T02:35:12,863 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-24T02:35:12,863 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:35:12,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:35:12,865 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:12,866 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:35:12,868 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'a67e7d17e593,38807,1732415667927' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-24T02:35:12,868 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-24T02:35:12,868 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1dbd57b4[Count = 0] remaining members to acquire global barrier 2024-11-24T02:35:12,869 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:12,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:12,878 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:12,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:12,878 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-24T02:35:12,879 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,879 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-24T02:35:12,879 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-24T02:35:12,879 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,879 INFO [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732415693359 highest: 1732415693395 on a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,879 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0:(num 1732415693359) roll requested 2024-11-24T02:35:12,888 DEBUG [master/a67e7d17e593:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732415677407 2024-11-24T02:35:12,896 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415712879, exclude list is [], retry=0 2024-11-24T02:35:12,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:35:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741888_1064 (size=950) 2024-11-24T02:35:12,901 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415712879 2024-11-24T02:35:12,901 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:35:12,901 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 is not closed yet, will try archiving it next time 2024-11-24T02:35:12,901 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.meta:.meta(num 1732415693377) roll requested 2024-11-24T02:35:12,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741857_1033 (size=1198) 2024-11-24T02:35:12,913 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415712901.meta, exclude list is [], retry=0 2024-11-24T02:35:12,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:35:12,917 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415712901.meta 2024-11-24T02:35:12,917 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:35:12,917 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta is not closed yet, will try archiving it next time 2024-11-24T02:35:12,917 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1:(num 1732415693395) roll requested 2024-11-24T02:35:12,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741858_1034 (size=93) 2024-11-24T02:35:12,918 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta not finished, retry = 0 2024-11-24T02:35:12,930 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415712917, exclude list is [], retry=0 2024-11-24T02:35:12,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:35:12,934 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415712917 2024-11-24T02:35:12,934 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:35:12,934 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 is not closed yet, will try archiving it next time 2024-11-24T02:35:12,934 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415679113 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415679113 2024-11-24T02:35:12,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741859_1035 (size=30490) 2024-11-24T02:35:12,935 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 not finished, retry = 0 2024-11-24T02:35:12,939 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-24T02:35:12,939 INFO [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732415712879 on a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:12,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:12,956 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-24T02:35:12,956 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'a67e7d17e593,38807,1732415667927' in zk 2024-11-24T02:35:12,986 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-24T02:35:12,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,986 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T02:35:12,987 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-24T02:35:12,987 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,987 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:35:12,987 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:35:12,988 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:35:12,988 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:35:12,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:12,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:35:12,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:12,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:12,994 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'a67e7d17e593,38807,1732415667927': 2024-11-24T02:35:12,994 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-24T02:35:12,994 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'a67e7d17e593,38807,1732415667927' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-24T02:35:12,994 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-24T02:35:12,994 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-24T02:35:12,994 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-24T02:35:12,994 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-24T02:35:13,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:35:13,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:35:13,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T02:35:13,003 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:13,003 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-24T02:35:13,003 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:35:13,003 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:13,003 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:35:13,004 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T02:35:13,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:35:13,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:35:13,004 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:13,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:13,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:13,004 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:35:13,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:13,005 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:13,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:13,005 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:35:13,006 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:13,006 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:13,020 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta 2024-11-24T02:35:13,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:13,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T02:35:13,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T02:35:13,024 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T02:35:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:13,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:35:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:13,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:35:13,024 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T02:35:13,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-24T02:35:13,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-24T02:35:13,024 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-11-24T02:35:13,024 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T02:35:13,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:35:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-24T02:35:13,033 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {a67e7d17e593:38807=1732415679113} newestTimestamps: {a67e7d17e593:38807=1732415693395} 2024-11-24T02:35:13,036 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: a67e7d17e593%2C38807%2C1732415667927.meta.1732415672505.meta 2024-11-24T02:35:13,036 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: a67e7d17e593%2C38807%2C1732415667927.meta.1732415712901.meta 2024-11-24T02:35:13,036 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 2024-11-24T02:35:13,036 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415712879 2024-11-24T02:35:13,036 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 2024-11-24T02:35:13,036 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415712917 2024-11-24T02:35:13,037 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta 2024-11-24T02:35:13,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741892_1068 (size=950) 2024-11-24T02:35:13,302 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/.tabledesc/.tableinfo.0000000002.908 2024-11-24T02:35:13,303 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/.tabledesc/.tableinfo.0000000003.950 2024-11-24T02:35:13,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732415677407 in 419 msec 2024-11-24T02:35:13,450 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732415677407 target: hdfs://localhost:40215/backupUT/backup_1732415708479/default/test-1732415677407 descriptor: 'test-1732415677407', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:35:13,451 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T02:35:13,457 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732415677407 2024-11-24T02:35:13,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741893_1069 (size=53) 2024-11-24T02:35:13,864 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732415677407 2024-11-24T02:35:13,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741894_1070 (size=364) 2024-11-24T02:35:14,284 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17324156774071 target: hdfs://localhost:40215/backupUT/backup_1732415708479/ns2/test-17324156774071 descriptor: 'ns2:test-17324156774071', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:35:14,284 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T02:35:14,287 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17324156774071 2024-11-24T02:35:14,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741895_1071 (size=50) 2024-11-24T02:35:14,695 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17324156774071 2024-11-24T02:35:14,712 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:40215/backupUT/.tmp/backup_1732415708479 from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359;hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 2024-11-24T02:35:14,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d02c1ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:14,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:14,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:14,715 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:14,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:14,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:14,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffc4cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:14,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:14,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:14,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:14,717 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:14,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f79c65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:14,718 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:35:14,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:35:14,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35ba807e 2024-11-24T02:35:14,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:35:14,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:35:14,725 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T02:35:14,725 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732415677407 2024-11-24T02:35:14,725 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732415677407,, stopping at row=test-1732415677407 ,, for max=2147483647 with caching=100 2024-11-24T02:35:14,726 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:14,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:14,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:14,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:14,730 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732415677407: test-1732415677407; 2024-11-24T02:35:14,731 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17324156774071 2024-11-24T02:35:14,731 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17324156774071,, stopping at row=ns2:test-17324156774071 ,, for max=2147483647 with caching=100 2024-11-24T02:35:14,733 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17324156774071: ns2:test-17324156774071; 2024-11-24T02:35:14,733 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-24T02:35:14,734 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/partitions_225aa236-aa45-4172-96d5-b9b9e9c6a302 2024-11-24T02:35:14,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741896_1072 (size=192) 2024-11-24T02:35:15,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:15,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:15,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,034 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-13541428994814289557.jar 2024-11-24T02:35:16,034 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,034 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,035 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,035 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,035 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:16,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:35:16,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:35:16,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:35:16,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:35:16,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:35:16,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:35:16,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:35:16,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:35:16,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:35:16,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:35:16,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:16,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:16,041 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: default:test-1732415677407,ns2:test-17324156774071 2024-11-24T02:35:16,041 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:35:16,042 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:35:16,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:16,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:16,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:16,042 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:16,899 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-6087526608821465186.jar 2024-11-24T02:35:16,908 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:16,934 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 for WAL files 2024-11-24T02:35:16,935 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 2024-11-24T02:35:16,935 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 for WAL files 2024-11-24T02:35:16,936 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 2024-11-24T02:35:17,031 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local103412286_0003_0b61540c-f3c0-446b-bc90-81c29413b424/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:35:17,031 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local103412286_0003_0b61540c-f3c0-446b-bc90-81c29413b424/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:35:17,107 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 for hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 (-9223372036854775808:9223372036854775807) length:30490 2024-11-24T02:35:17,131 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-24T02:35:17,131 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-24T02:35:17,170 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 for hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 (-9223372036854775808:9223372036854775807) length:1198 2024-11-24T02:35:17,173 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-24T02:35:17,173 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-24T02:35:17,248 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1044944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,248 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:17,248 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:17,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:17,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:17,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:17,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@737eaf74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:17,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:17,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:17,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2977b244, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,252 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:17,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:17,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:17,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:17,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17324156774071', row='row-t20', locateType=CURRENT is [region=ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:35:17,257 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-24T02:35:17,257 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:35:17,257 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:17,257 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,258 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-24T02:35:17,258 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:35:17,263 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/backupUT/.tmp/backup_1732415708479/_temporary/0/_temporary/attempt_local103412286_0003_r_000000_0/ns2/test-17324156774071/f/f7a9bfb58ff84572b5229ecddd816666 is 38, key is row-t20/f:q1/1732415707124/Put/seqid=0 2024-11-24T02:35:17,265 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:35:17,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741897_1073 (size=5274) 2024-11-24T02:35:17,708 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d0bcc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,708 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:17,708 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:17,709 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:17,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:17,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:17,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159d3336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:17,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:17,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,710 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:17,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a679eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,711 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:17,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:17,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:17,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:17,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732415677407', row='row-test-1732415677407-1-0', locateType=CURRENT is [region=test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., hostname=a67e7d17e593,38807,1732415667927, seqNum=107] 2024-11-24T02:35:17,715 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-24T02:35:17,715 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:35:17,715 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,716 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:17,716 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732415677407-1-0] 2024-11-24T02:35:17,716 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:35:17,727 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6a3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,728 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:17,728 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:17,729 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:17,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:17,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:17,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e65559d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:17,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:17,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,730 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:17,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74270c23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:17,732 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:17,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:17,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:17,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:17,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732415677407', row='row-test-1732415677407-3-0', locateType=CURRENT is [region=test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., hostname=a67e7d17e593,38807,1732415667927, seqNum=107] 2024-11-24T02:35:17,736 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-24T02:35:17,736 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:35:17,736 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,736 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:17,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:17,736 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732415677407-3-0] 2024-11-24T02:35:17,736 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:35:17,748 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/backupUT/.tmp/backup_1732415708479/_temporary/0/_temporary/attempt_local103412286_0003_r_000001_0/default/test-1732415677407/f/4a8efe77ff664b709a922f556d46cd93 is 59, key is row-test-1732415677407-1-10/f:q1/1732415706753/Put/seqid=0 2024-11-24T02:35:17,750 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:35:17,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741898_1074 (size=10769) 2024-11-24T02:35:18,155 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/backupUT/.tmp/backup_1732415708479/_temporary/0/_temporary/attempt_local103412286_0003_r_000001_0/default/test-1732415677407/mob/c49ad642a6874e8f9c24981b321bd550 is 63, key is row-test-1732415677407-3-100/mob:q1/1732415707081/Put/seqid=0 2024-11-24T02:35:18,204 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:35:18,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741899_1075 (size=11810) 2024-11-24T02:35:19,079 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:40215/backupUT 2024-11-24T02:35:19,079 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732415708479 2024-11-24T02:35:19,079 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-24T02:35:19,095 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:40215/backupUT/.tmp/backup_1732415708479, hdfs://localhost:40215/backupUT] 2024-11-24T02:35:19,160 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:19,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741900_1076 (size=10769) 2024-11-24T02:35:19,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741901_1077 (size=5274) 2024-11-24T02:35:20,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741902_1078 (size=11810) 2024-11-24T02:35:20,767 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-24T02:35:20,772 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732415708479 2024-11-24T02:35:20,772 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27853 bytes copied." 2024-11-24T02:35:20,773 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1762143613_0004 completed: true true 2024-11-24T02:35:20,775 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3825503 FILE: Number of bytes written=6195357 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231464 HDFS: Number of bytes written=2433179 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=250 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=340 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27853 Bytes Copied=27853 Bytes Expected=27853 Files Copied=4 DIR_COPY=8 2024-11-24T02:35:20,775 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:40215/backupUT/.tmp/backup_1732415708479 to hdfs://localhost:40215/backupUT finished. 2024-11-24T02:35:20,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:test-1732415677407:', locateType=CURRENT is [region=backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:35:20,791 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732415708479 2024-11-24T02:35:20,795 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732415687521} 2024-11-24T02:35:20,795 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-24T02:35:20,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741903_1079 (size=384) 2024-11-24T02:35:21,201 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:40215/backupUT/backup_1732415708479/.backup.manifest 2024-11-24T02:35:21,201 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732415708479 finished: type=INCREMENTAL,tablelist=test-1732415677407;ns2:test-17324156774071,targetRootDir=hdfs://localhost:40215/backupUT,startts=1732415712753,completets=1732415720791,bytescopied=0 2024-11-24T02:35:21,201 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T02:35:21,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-24T02:35:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-24T02:35:21,216 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-24T02:35:21,219 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T02:35:21,224 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732415708479 completed. 2024-11-24T02:35:21,224 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-24T02:35:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:21,248 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:40215/backupUT/backup_1732415708479/.backup.manifest 2024-11-24T02:35:21,251 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732415708479/.backup.manifest 2024-11-24T02:35:21,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732415677407 from 'test-1732415677407', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732415677407', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:35:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732415677407 2024-11-24T02:35:21,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741904_1080 (size=908) 2024-11-24T02:35:21,669 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/.tabledesc/.tableinfo.0000000003.950 2024-11-24T02:35:21,670 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/.tabledesc/.tableinfo.0000000004.908 2024-11-24T02:35:21,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732415677407}] 2024-11-24T02:35:21,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE}] 2024-11-24T02:35:21,677 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE 2024-11-24T02:35:21,678 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=CLOSING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:35:21,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE because future has completed 2024-11-24T02:35:21,680 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:35:21,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:35:21,834 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:21,834 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:35:21,835 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing a9fcfc65d96498f86ae4291b18ffd315, disabling compactions & flushes 2024-11-24T02:35:21,835 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:21,835 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:21,835 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. after waiting 0 ms 2024-11-24T02:35:21,835 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:21,848 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-24T02:35:21,848 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:35:21,848 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:21,848 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for a9fcfc65d96498f86ae4291b18ffd315: Waiting for close lock at 1732415721835Running coprocessor pre-close hooks at 1732415721835Disabling compacts and flushes for region at 1732415721835Disabling writes for close at 1732415721835Writing region close event to WAL at 1732415721841 (+6 ms)Running coprocessor post-close hooks at 1732415721848 (+7 ms)Closed at 1732415721848 2024-11-24T02:35:21,848 WARN [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: a9fcfc65d96498f86ae4291b18ffd315 to self. 2024-11-24T02:35:21,850 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:21,851 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=CLOSED 2024-11-24T02:35:21,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:35:21,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T02:35:21,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 in 173 msec 2024-11-24T02:35:21,856 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE; state=CLOSED, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=true 2024-11-24T02:35:22,007 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:35:22,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE because future has completed 2024-11-24T02:35:22,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:35:22,176 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:22,176 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => a9fcfc65d96498f86ae4291b18ffd315, NAME => 'test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:35:22,176 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:35:22,177 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732415677407 a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,177 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:35:22,177 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,177 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,179 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,180 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9fcfc65d96498f86ae4291b18ffd315 columnFamilyName f 2024-11-24T02:35:22,180 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:35:22,190 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e 2024-11-24T02:35:22,196 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 2024-11-24T02:35:22,196 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(327): Store=a9fcfc65d96498f86ae4291b18ffd315/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:35:22,197 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,199 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9fcfc65d96498f86ae4291b18ffd315 columnFamilyName f2 2024-11-24T02:35:22,199 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:35:22,199 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(327): Store=a9fcfc65d96498f86ae4291b18ffd315/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:35:22,200 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,201 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9fcfc65d96498f86ae4291b18ffd315 columnFamilyName mob 2024-11-24T02:35:22,201 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:35:22,209 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6 2024-11-24T02:35:22,214 DEBUG [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c 2024-11-24T02:35:22,214 INFO [StoreOpener-a9fcfc65d96498f86ae4291b18ffd315-1 {}] regionserver.HStore(327): Store=a9fcfc65d96498f86ae4291b18ffd315/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:35:22,214 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,215 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,216 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,217 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,217 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,218 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732415677407 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T02:35:22,219 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,220 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened a9fcfc65d96498f86ae4291b18ffd315; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11126713280, jitterRate=0.03625592589378357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T02:35:22,221 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:22,225 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for a9fcfc65d96498f86ae4291b18ffd315: Running coprocessor pre-open hook at 1732415722177Writing region info on filesystem at 1732415722177Initializing all the Stores at 1732415722178 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415722178Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415722179 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415722179Cleaning up temporary data from old regions at 1732415722217 (+38 ms)Running coprocessor post-open hooks at 1732415722221 (+4 ms)Region opened successfully at 1732415722225 (+4 ms) 2024-11-24T02:35:22,226 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., pid=46, masterSystemTime=1732415722166 2024-11-24T02:35:22,228 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:22,228 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:22,229 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=a9fcfc65d96498f86ae4291b18ffd315, regionState=OPEN, openSeqNum=338, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:35:22,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:35:22,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-24T02:35:22,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure a9fcfc65d96498f86ae4291b18ffd315, server=a67e7d17e593,38807,1732415667927 in 220 msec 2024-11-24T02:35:22,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-24T02:35:22,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732415677407, region=a9fcfc65d96498f86ae4291b18ffd315, REOPEN/MOVE in 557 msec 2024-11-24T02:35:22,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T02:35:22,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732415677407 in 563 msec 2024-11-24T02:35:22,240 DEBUG [PEWorker-4 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1732415677407 2024-11-24T02:35:22,246 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 2024-11-24T02:35:22,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732415677407 in 995 msec 2024-11-24T02:35:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:35:22,251 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732415677407 completed 2024-11-24T02:35:22,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,286 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T02:35:22,291 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732415722251 2024-11-24T02:35:22,293 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1732415708479} 2024-11-24T02:35:22,293 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732415687521} 2024-11-24T02:35:22,293 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-24T02:35:22,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,334 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1732415677407 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1732415677407 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:35:22,337 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732415722251,startts=0,failedts=1732415722335,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1732415677407 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-24T02:35:22,337 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-24T02:35:22,339 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-24T02:35:22,339 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T02:35:22,340 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-24T02:35:22,341 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-24T02:35:22,342 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T02:35:22,345 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732415722251 failed. 2024-11-24T02:35:22,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:22,365 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1732415677407, ns2:test-17324156774071, ns3:test-17324156774072, ns4:test-17324156774073] 2024-11-24T02:35:22,365 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T02:35:22,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T02:35:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415722370 (current time:1732415722370). 2024-11-24T02:35:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:35:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-24T02:35:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e57109d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:22,371 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:22,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:22,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:22,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d147a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:22,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:22,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:22,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:22,373 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:22,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d09940e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:22,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:22,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:22,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:22,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:35:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:22,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:35:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T02:35:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-24T02:35:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T02:35:22,380 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:35:22,382 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:35:22,384 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:35:22,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741905_1081 (size=52) 2024-11-24T02:35:22,391 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:35:22,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea}] 2024-11-24T02:35:22,392 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T02:35:22,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-11-24T02:35:22,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:35:22,545 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing 6df51c7a7a1591e81897afbbac0da0ea 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-24T02:35:22,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/6214c14cce634e038861bdb939ed2400 is 158, key is trslm:hdfs://localhost:40215/backupUT\x00ns2:test-17324156774071/meta:log-roll-map/1732415720781/Put/seqid=0 2024-11-24T02:35:22,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741906_1082 (size=5745) 2024-11-24T02:35:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T02:35:22,958 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/6214c14cce634e038861bdb939ed2400 2024-11-24T02:35:22,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/ca5f74340e0d45e5b18e062fa6ed1c77 is 1245, key is session:backup_1732415722251/session:context/1732415722341/Put/seqid=0 2024-11-24T02:35:22,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741907_1083 (size=6682) 2024-11-24T02:35:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T02:35:23,379 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/ca5f74340e0d45e5b18e062fa6ed1c77 2024-11-24T02:35:23,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/6214c14cce634e038861bdb939ed2400 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/6214c14cce634e038861bdb939ed2400 2024-11-24T02:35:23,398 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/6214c14cce634e038861bdb939ed2400, entries=4, sequenceid=31, filesize=5.6 K 2024-11-24T02:35:23,399 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/ca5f74340e0d45e5b18e062fa6ed1c77 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/ca5f74340e0d45e5b18e062fa6ed1c77 2024-11-24T02:35:23,408 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/ca5f74340e0d45e5b18e062fa6ed1c77, entries=3, sequenceid=31, filesize=6.5 K 2024-11-24T02:35:23,409 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 6df51c7a7a1591e81897afbbac0da0ea in 865ms, sequenceid=31, compaction requested=true 2024-11-24T02:35:23,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for 6df51c7a7a1591e81897afbbac0da0ea: 2024-11-24T02:35:23,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. for snapshot_backup_system completed. 2024-11-24T02:35:23,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.' region-info for snapshot=snapshot_backup_system 2024-11-24T02:35:23,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:35:23,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/09439360a3384a58b0af1fd4ddfc50f5, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/6214c14cce634e038861bdb939ed2400] hfiles 2024-11-24T02:35:23,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/09439360a3384a58b0af1fd4ddfc50f5 for snapshot=snapshot_backup_system 2024-11-24T02:35:23,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/6214c14cce634e038861bdb939ed2400 for snapshot=snapshot_backup_system 2024-11-24T02:35:23,411 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/bc9a95a157c24f92aeeba767a22b2c63, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/ca5f74340e0d45e5b18e062fa6ed1c77] hfiles 2024-11-24T02:35:23,411 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d for snapshot=snapshot_backup_system 2024-11-24T02:35:23,411 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/bc9a95a157c24f92aeeba767a22b2c63 for snapshot=snapshot_backup_system 2024-11-24T02:35:23,411 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/ca5f74340e0d45e5b18e062fa6ed1c77 for snapshot=snapshot_backup_system 2024-11-24T02:35:23,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741908_1084 (size=251) 2024-11-24T02:35:23,463 DEBUG [master/a67e7d17e593:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732415677407 2024-11-24T02:35:23,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741909_1085 (size=950) 2024-11-24T02:35:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T02:35:23,817 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:35:23,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-24T02:35:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-11-24T02:35:23,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:23,819 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:23,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-24T02:35:23,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea in 1.4280 sec 2024-11-24T02:35:23,821 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:35:23,821 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:35:23,822 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:35:23,822 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-24T02:35:23,823 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-24T02:35:23,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741910_1086 (size=888) 2024-11-24T02:35:23,874 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/.tabledesc/.tableinfo.0000000004.908 2024-11-24T02:35:23,874 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/.tabledesc/.tableinfo.0000000005.950 2024-11-24T02:35:23,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732415677407 in 412 msec 2024-11-24T02:35:24,234 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:35:24,242 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:35:24,242 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_backup_system 2024-11-24T02:35:24,243 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:35:24,244 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-24T02:35:24,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8660 sec 2024-11-24T02:35:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-24T02:35:24,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-24T02:35:24,534 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732415722346 started at 1732415724533. 2024-11-24T02:35:24,539 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732415722346 has been started. 2024-11-24T02:35:24,547 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-24T02:35:24,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-24T02:35:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-24T02:35:24,549 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-24T02:35:24,549 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-24T02:35:24,549 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-24T02:35:24,549 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-24T02:35:24,550 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,550 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:24,618 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T02:35:24,618 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T02:35:24,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:35:24,619 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,619 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-24T02:35:24,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:24,621 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-24T02:35:24,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:24,622 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-24T02:35:24,623 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-24T02:35:24,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-24T02:35:24,623 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-24T02:35:24,623 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-24T02:35:24,623 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-24T02:35:24,623 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-24T02:35:24,623 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'a67e7d17e593,38807,1732415667927' joining acquired barrier for procedure (rolllog) in zk 2024-11-24T02:35:24,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,634 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:24,634 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,634 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:35:24,634 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:35:24,634 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-24T02:35:24,634 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-24T02:35:24,634 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:35:24,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:35:24,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:24,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,636 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:35:24,636 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'a67e7d17e593,38807,1732415667927' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-24T02:35:24,636 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@288a3353[Count = 0] remaining members to acquire global barrier 2024-11-24T02:35:24,636 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-24T02:35:24,636 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:24,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:24,642 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:24,642 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:24,642 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-24T02:35:24,643 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,643 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-24T02:35:24,643 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-24T02:35:24,643 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,643 INFO [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732415712879 highest: 1732415712917 on a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,643 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0:(num 1732415712879) roll requested 2024-11-24T02:35:24,659 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643, exclude list is [], retry=0 2024-11-24T02:35:24,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:35:24,663 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415712879 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 2024-11-24T02:35:24,663 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:35:24,663 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415712879 is not closed yet, will try archiving it next time 2024-11-24T02:35:24,663 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.meta:.meta(num 1732415712901) roll requested 2024-11-24T02:35:24,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741889_1065 (size=93) 2024-11-24T02:35:24,664 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415712879 not finished, retry = 0 2024-11-24T02:35:24,675 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415724663.meta, exclude list is [], retry=0 2024-11-24T02:35:24,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:35:24,678 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415712901.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415724663.meta 2024-11-24T02:35:24,679 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:35:24,679 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415712901.meta is not closed yet, will try archiving it next time 2024-11-24T02:35:24,679 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1:(num 1732415712917) roll requested 2024-11-24T02:35:24,680 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415712901.meta not finished, retry = 0 2024-11-24T02:35:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741890_1066 (size=1903) 2024-11-24T02:35:24,690 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679, exclude list is [], retry=0 2024-11-24T02:35:24,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:35:24,694 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415712917 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679 2024-11-24T02:35:24,694 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:35:24,694 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415712917 is not closed yet, will try archiving it next time 2024-11-24T02:35:24,694 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415693395 2024-11-24T02:35:24,695 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415712917 not finished, retry = 0 2024-11-24T02:35:24,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741891_1067 (size=4927) 2024-11-24T02:35:24,703 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-24T02:35:24,704 INFO [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732415724643 on a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:24,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:24,718 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-24T02:35:24,718 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'a67e7d17e593,38807,1732415667927' in zk 2024-11-24T02:35:24,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,734 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-24T02:35:24,734 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T02:35:24,734 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-24T02:35:24,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:35:24,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:35:24,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:35:24,735 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:35:24,735 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:24,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:35:24,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:24,737 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'a67e7d17e593,38807,1732415667927': 2024-11-24T02:35:24,738 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-24T02:35:24,738 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-24T02:35:24,738 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'a67e7d17e593,38807,1732415667927' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-24T02:35:24,738 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-24T02:35:24,738 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-24T02:35:24,738 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-24T02:35:24,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T02:35:24,750 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:35:24,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:35:24,751 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T02:35:24,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:35:24,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:35:24,751 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:24,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:35:24,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:24,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,753 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:35:24,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:35:24,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T02:35:24,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:35:24,759 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T02:35:24,759 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T02:35:24,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-24T02:35:24,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-24T02:35:24,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:35:24,760 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T02:35:24,760 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:35:24,765 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415712879 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415712879 2024-11-24T02:35:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-24T02:35:24,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-24T02:35:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415724774 (current time:1732415724774). 2024-11-24T02:35:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:35:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732415724773_backup_system_bulk VERSION not specified, setting to 2 2024-11-24T02:35:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60080212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:24,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:24,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:24,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:24,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ccde8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:24,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:24,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:24,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:24,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:24,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22380f0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:24,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:24,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:24,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:24,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:35:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:24,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:35:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-24T02:35:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-24T02:35:24,782 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:35:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:35:24,783 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:35:24,785 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:35:24,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741914_1090 (size=76) 2024-11-24T02:35:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:35:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:35:25,161 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:35:25,194 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:35:25,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 48740e06c1a2c00320ec32fe075fe704}] 2024-11-24T02:35:25,196 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:35:25,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-24T02:35:25,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:35:25,351 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 48740e06c1a2c00320ec32fe075fe704: 2024-11-24T02:35:25,351 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. for snapshot_1732415724773_backup_system_bulk completed. 2024-11-24T02:35:25,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704.' region-info for snapshot=snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:25,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:35:25,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T02:35:25,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T02:35:25,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741915_1091 (size=61) 2024-11-24T02:35:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:35:25,765 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:35:25,765 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-24T02:35:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-24T02:35:25,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732415724773_backup_system_bulk on region 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:35:25,767 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:35:25,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-24T02:35:25,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 48740e06c1a2c00320ec32fe075fe704 in 576 msec 2024-11-24T02:35:25,773 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:35:25,775 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:35:25,776 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:35:25,776 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:25,777 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:25,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741916_1092 (size=702) 2024-11-24T02:35:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:35:26,015 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 50af115aeed7c572d01a2d7d6548f037, had cached 0 bytes from a total of 8413 2024-11-24T02:35:26,195 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:35:26,202 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:35:26,203 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415724773_backup_system_bulk to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:26,204 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:35:26,204 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-24T02:35:26,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732415724773_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.4230 sec 2024-11-24T02:35:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:35:26,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-24T02:35:26,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } 2024-11-24T02:35:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415726932 (current time:1732415726932). 2024-11-24T02:35:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:35:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732415726932_default_test-1732415677407 VERSION not specified, setting to 2 2024-11-24T02:35:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e53892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:26,934 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:26,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314025e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:26,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:26,936 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:26,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29e51328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:26,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:26,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:26,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:26,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:35:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:26,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:35:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } 2024-11-24T02:35:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-24T02:35:26,945 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:35:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:35:26,946 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:35:26,950 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:35:26,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741917_1093 (size=84) 2024-11-24T02:35:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:35:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:35:27,361 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:35:27,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a9fcfc65d96498f86ae4291b18ffd315}] 2024-11-24T02:35:27,365 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:27,379 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T02:35:27,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-24T02:35:27,520 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:27,520 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a9fcfc65d96498f86ae4291b18ffd315: 2024-11-24T02:35:27,520 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. for snapshot_1732415726932_default_test-1732415677407 completed. 2024-11-24T02:35:27,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315.' region-info for snapshot=snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:27,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:35:27,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e] hfiles 2024-11-24T02:35:27,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 for snapshot=snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:27,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e for snapshot=snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:27,522 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T02:35:27,522 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6] hfiles 2024-11-24T02:35:27,522 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c for snapshot=snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:27,522 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6 for snapshot=snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:27,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741918_1094 (size=224) 2024-11-24T02:35:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:35:27,933 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:35:27,933 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-24T02:35:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-24T02:35:27,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732415726932_default_test-1732415677407 on region a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:27,935 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:27,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-24T02:35:27,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a9fcfc65d96498f86ae4291b18ffd315 in 575 msec 2024-11-24T02:35:27,940 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:35:27,942 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:35:27,943 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-24T02:35:27,943 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-24T02:35:27,944 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:35:27,944 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-24T02:35:27,944 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:35:27,945 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-24T02:35:27,945 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:35:27,947 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79/mob/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315] hfiles 2024-11-24T02:35:27,947 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79/mob/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741919_1095 (size=168) 2024-11-24T02:35:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:35:28,357 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:35:28,357 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:28,360 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:28,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741920_1096 (size=1347) 2024-11-24T02:35:28,780 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:35:28,791 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:35:28,791 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415726932_default_test-1732415677407 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:28,793 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:35:28,793 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-24T02:35:28,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732415726932_default_test-1732415677407 table=test-1732415677407 type=FLUSH ttl=0 } in 1.8510 sec 2024-11-24T02:35:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:35:29,093 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732415677407 completed 2024-11-24T02:35:29,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } 2024-11-24T02:35:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415729095 (current time:1732415729095). 2024-11-24T02:35:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:35:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732415729093_ns2_test-17324156774071 VERSION not specified, setting to 2 2024-11-24T02:35:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2155357a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:29,099 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:29,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:29,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:29,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cea2fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:29,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:29,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:29,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:29,102 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:29,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2b19e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:29,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:29,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:29,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:29,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:35:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:29,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:35:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } 2024-11-24T02:35:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-24T02:35:29,111 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:35:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:35:29,112 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:35:29,115 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:35:29,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741921_1097 (size=86) 2024-11-24T02:35:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:35:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:35:29,522 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:35:29,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 50af115aeed7c572d01a2d7d6548f037}] 2024-11-24T02:35:29,522 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:35:29,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-24T02:35:29,676 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:35:29,677 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 50af115aeed7c572d01a2d7d6548f037 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-24T02:35:29,688 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/.tmp/f/8d78355b3323442fa3b7a801ae673cb9 is 38, key is row-t20/f:q1/1732415707124/Put/seqid=0 2024-11-24T02:35:29,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741922_1098 (size=5134) 2024-11-24T02:35:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:35:30,095 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/.tmp/f/8d78355b3323442fa3b7a801ae673cb9 2024-11-24T02:35:30,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/.tmp/f/8d78355b3323442fa3b7a801ae673cb9 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/8d78355b3323442fa3b7a801ae673cb9 2024-11-24T02:35:30,115 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/8d78355b3323442fa3b7a801ae673cb9, entries=5, sequenceid=111, filesize=5.0 K 2024-11-24T02:35:30,116 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 50af115aeed7c572d01a2d7d6548f037 in 440ms, sequenceid=111, compaction requested=false 2024-11-24T02:35:30,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 50af115aeed7c572d01a2d7d6548f037: 2024-11-24T02:35:30,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. for snapshot_1732415729093_ns2_test-17324156774071 completed. 2024-11-24T02:35:30,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037.' region-info for snapshot=snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:30,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:35:30,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/8d78355b3323442fa3b7a801ae673cb9] hfiles 2024-11-24T02:35:30,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 for snapshot=snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:30,117 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/8d78355b3323442fa3b7a801ae673cb9 for snapshot=snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:30,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741923_1099 (size=130) 2024-11-24T02:35:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:35:30,525 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:35:30,525 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-24T02:35:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-24T02:35:30,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732415729093_ns2_test-17324156774071 on region 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:35:30,527 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:35:30,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-24T02:35:30,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 50af115aeed7c572d01a2d7d6548f037 in 1.0070 sec 2024-11-24T02:35:30,534 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:35:30,537 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:35:30,539 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:35:30,539 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:30,540 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:30,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741924_1100 (size=496) 2024-11-24T02:35:30,952 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:35:30,963 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:35:30,963 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415729093_ns2_test-17324156774071 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:30,965 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:35:30,965 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-24T02:35:30,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732415729093_ns2_test-17324156774071 table=ns2:test-17324156774071 type=FLUSH ttl=0 } in 1.8560 sec 2024-11-24T02:35:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:35:31,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17324156774071 completed 2024-11-24T02:35:31,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } 2024-11-24T02:35:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415731262 (current time:1732415731262). 2024-11-24T02:35:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:35:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732415731261_ns4_test-17324156774073 VERSION not specified, setting to 2 2024-11-24T02:35:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ad81c23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:31,265 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:31,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:31,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:31,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407d2ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:31,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:31,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:31,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:31,266 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:31,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40cd432a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:31,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:31,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:31,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:31,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:35:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:31,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:35:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } 2024-11-24T02:35:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-24T02:35:31,273 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:35:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T02:35:31,274 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:35:31,277 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:35:31,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741925_1101 (size=86) 2024-11-24T02:35:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T02:35:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T02:35:31,685 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:35:31,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a605441c1cb1d296fb15e4b88f05f9c}] 2024-11-24T02:35:31,688 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:35:31,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-24T02:35:31,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:35:31,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 1a605441c1cb1d296fb15e4b88f05f9c: 2024-11-24T02:35:31,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. for snapshot_1732415731261_ns4_test-17324156774073 completed. 2024-11-24T02:35:31,844 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c.' region-info for snapshot=snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:31,844 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:35:31,844 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T02:35:31,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741926_1102 (size=52) 2024-11-24T02:35:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T02:35:32,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:35:32,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-24T02:35:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-24T02:35:32,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732415731261_ns4_test-17324156774073 on region 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:35:32,257 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:35:32,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-24T02:35:32,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1a605441c1cb1d296fb15e4b88f05f9c in 574 msec 2024-11-24T02:35:32,262 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:35:32,263 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:35:32,264 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:35:32,264 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:32,265 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:32,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741927_1103 (size=443) 2024-11-24T02:35:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T02:35:32,678 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:35:32,685 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:35:32,686 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415731261_ns4_test-17324156774073 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:32,687 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:35:32,687 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-24T02:35:32,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732415731261_ns4_test-17324156774073 table=ns4:test-17324156774073 type=FLUSH ttl=0 } in 1.4160 sec 2024-11-24T02:35:32,894 INFO [master/a67e7d17e593:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T02:35:32,894 INFO [master/a67e7d17e593:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T02:35:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T02:35:33,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17324156774073 completed 2024-11-24T02:35:33,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } 2024-11-24T02:35:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415733425 (current time:1732415733425). 2024-11-24T02:35:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:35:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732415733423_ns3_test-17324156774072 VERSION not specified, setting to 2 2024-11-24T02:35:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b55b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:33,429 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:33,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:33,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:33,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cff31d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:33,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:33,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:33,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:33,431 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:33,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70512fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:33,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:33,433 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6df51c7a7a1591e81897afbbac0da0ea, had cached 0 bytes from a total of 28925 2024-11-24T02:35:33,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:33,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:33,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:35:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:33,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:35:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } 2024-11-24T02:35:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-24T02:35:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:35:33,438 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:35:33,439 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:35:33,442 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:35:33,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741928_1104 (size=86) 2024-11-24T02:35:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:35:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:35:33,851 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:35:33,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure aca5da70eca26b7decb49ebc97e25da8}] 2024-11-24T02:35:33,852 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:35:34,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-24T02:35:34,004 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:35:34,004 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for aca5da70eca26b7decb49ebc97e25da8: 2024-11-24T02:35:34,004 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. for snapshot_1732415733423_ns3_test-17324156774072 completed. 2024-11-24T02:35:34,004 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8.' region-info for snapshot=snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:34,004 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:35:34,004 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-24T02:35:34,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741929_1105 (size=52) 2024-11-24T02:35:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:35:34,412 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:35:34,412 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-24T02:35:34,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-24T02:35:34,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732415733423_ns3_test-17324156774072 on region aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:35:34,414 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:35:34,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-24T02:35:34,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure aca5da70eca26b7decb49ebc97e25da8 in 566 msec 2024-11-24T02:35:34,422 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:35:34,423 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:35:34,424 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:35:34,424 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:34,426 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:34,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741930_1106 (size=443) 2024-11-24T02:35:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:35:34,842 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:35:34,850 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:35:34,850 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_1732415733423_ns3_test-17324156774072 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:34,852 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:35:34,852 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-24T02:35:34,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732415733423_ns3_test-17324156774072 table=ns3:test-17324156774072 type=FLUSH ttl=0 } in 1.4160 sec 2024-11-24T02:35:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:35:35,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17324156774072 completed 2024-11-24T02:35:35,593 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732415722346 2024-11-24T02:35:35,593 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-24T02:35:35,593 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-24T02:35:35,593 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732415722346_backup:system_bulk 2024-11-24T02:35:35,593 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732415724773_backup_system_bulk to hdfs://localhost:40215/backupUT/backup_1732415722346/backup/system_bulk/ 2024-11-24T02:35:35,594 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T02:35:35,627 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:40215, inputRoot=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:35:35,627 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_10852338_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40215/backupUT/backup_1732415722346/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40215/backupUT/backup_1732415722346/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:35,629 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T02:35:35,633 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415724773_backup_system_bulk to hdfs://localhost:40215/backupUT/backup_1732415722346/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:35,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741932_1108 (size=76) 2024-11-24T02:35:35,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741931_1107 (size=702) 2024-11-24T02:35:36,049 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,050 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,051 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-13943623375283964453.jar 2024-11-24T02:35:36,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:36,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:36,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:35:36,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:35:36,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:35:36,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:35:36,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:35:36,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:35:36,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:35:36,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:35:36,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:35:36,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:35:36,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:36,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:36,916 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:36,916 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:36,916 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:36,916 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:36,916 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:36,950 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:36,973 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732415724773_backup_system_bulk' hfile list 2024-11-24T02:35:37,048 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1848292142_0005_fe52aad5-a4c1-4e17-ae11-4c9496de28c5/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:35:37,048 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1848292142_0005_fe52aad5-a4c1-4e17-ae11-4c9496de28c5/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:35:38,085 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T02:35:38,086 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T02:35:38,090 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:38,091 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732415724773_backup_system_bulk finished. 2024-11-24T02:35:38,091 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732415722346_test-1732415677407 2024-11-24T02:35:38,091 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732415726932_default_test-1732415677407 to hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/ 2024-11-24T02:35:38,091 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T02:35:38,119 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:40215, inputRoot=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:35:38,119 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_10852338_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/.hbase-snapshot/.tmp/snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:38,121 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T02:35:38,128 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415726932_default_test-1732415677407 to hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/.hbase-snapshot/.tmp/snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:38,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741933_1109 (size=1347) 2024-11-24T02:35:38,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741934_1110 (size=84) 2024-11-24T02:35:38,543 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:38,543 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:38,544 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-17014013229051832157.jar 2024-11-24T02:35:39,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:39,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:39,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:35:39,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:35:39,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:35:39,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:35:39,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:35:39,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:35:39,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:35:39,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:35:39,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:35:39,414 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:35:39,414 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:39,414 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:39,414 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:39,414 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:39,415 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:39,415 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:39,415 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:39,449 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:39,475 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732415726932_default_test-1732415677407' hfile list 2024-11-24T02:35:39,477 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-11-24T02:35:39,563 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local608256727_0006_c42ce544-3847-43fe-8108-33380cfef208/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:35:39,563 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local608256727_0006_c42ce544-3847-43fe-8108-33380cfef208/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:35:39,633 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-24T02:35:39,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741935_1111 (size=13760) 2024-11-24T02:35:40,070 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6 output=hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6 2024-11-24T02:35:40,071 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-24T02:35:40,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741936_1112 (size=10796) 2024-11-24T02:35:40,500 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e output=hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e 2024-11-24T02:35:40,501 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-24T02:35:40,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741937_1113 (size=8413) 2024-11-24T02:35:40,935 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 output=hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 2024-11-24T02:35:40,935 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-11-24T02:35:40,951 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79/mob/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315 to hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79/mob/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315" truncated to max limit (512 characters) 2024-11-24T02:35:40,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741938_1114 (size=5591) 2024-11-24T02:35:41,361 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79/mob/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315 output=hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79/mob/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:35:41,361 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.013M/sec 2024-11-24T02:35:41,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741939_1115 (size=5484) 2024-11-24T02:35:41,800 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c output=hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c 2024-11-24T02:35:41,800 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-24T02:35:42,602 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T02:35:42,604 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T02:35:42,617 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:42,617 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732415726932_default_test-1732415677407 finished. 2024-11-24T02:35:42,617 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732415722346_ns2:test-17324156774071 2024-11-24T02:35:42,617 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732415729093_ns2_test-17324156774071 to hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/ 2024-11-24T02:35:42,617 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T02:35:42,645 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:40215, inputRoot=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:35:42,645 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_10852338_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/.hbase-snapshot/.tmp/snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:42,647 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T02:35:42,652 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415729093_ns2_test-17324156774071 to hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/.hbase-snapshot/.tmp/snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:42,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741941_1117 (size=496) 2024-11-24T02:35:42,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741940_1116 (size=86) 2024-11-24T02:35:43,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,069 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-15100898598379659270.jar 2024-11-24T02:35:43,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,929 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,929 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,929 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,929 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,929 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,930 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:43,930 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:43,930 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:35:43,930 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:35:43,931 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:35:43,931 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:35:43,931 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:35:43,931 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:35:43,931 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:35:43,932 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:35:43,932 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:35:43,932 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:35:43,932 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:43,932 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:43,933 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:43,933 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:43,933 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:43,933 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:43,933 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:43,967 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:43,990 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732415729093_ns2_test-17324156774071' hfile list 2024-11-24T02:35:43,992 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-11-24T02:35:44,071 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local344342424_0007_50775c42-cb05-4771-9824-87f665a86ba4/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:35:44,071 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local344342424_0007_50775c42-cb05-4771-9824-87f665a86ba4/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:35:44,131 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-24T02:35:44,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741942_1118 (size=8413) 2024-11-24T02:35:44,556 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 output=hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 2024-11-24T02:35:44,557 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-11-24T02:35:44,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741943_1119 (size=5134) 2024-11-24T02:35:44,994 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/8d78355b3323442fa3b7a801ae673cb9 output=hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/8d78355b3323442fa3b7a801ae673cb9 2024-11-24T02:35:44,994 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-11-24T02:35:45,107 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T02:35:45,108 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T02:35:45,116 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:45,116 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732415729093_ns2_test-17324156774071 finished. 2024-11-24T02:35:45,116 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732415722346_ns4:test-17324156774073 2024-11-24T02:35:45,116 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732415731261_ns4_test-17324156774073 to hdfs://localhost:40215/backupUT/backup_1732415722346/ns4/test-17324156774073/ 2024-11-24T02:35:45,116 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T02:35:45,141 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:40215, inputRoot=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:35:45,141 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_10852338_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40215/backupUT/backup_1732415722346/ns4/test-17324156774073, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40215/backupUT/backup_1732415722346/ns4/test-17324156774073/.hbase-snapshot/.tmp/snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:45,143 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T02:35:45,146 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415731261_ns4_test-17324156774073 to hdfs://localhost:40215/backupUT/backup_1732415722346/ns4/test-17324156774073/.hbase-snapshot/.tmp/snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:45,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741945_1121 (size=86) 2024-11-24T02:35:45,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741944_1120 (size=443) 2024-11-24T02:35:45,561 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:45,562 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:45,562 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-10090367128171046982.jar 2024-11-24T02:35:46,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,425 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,426 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:46,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:46,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:35:46,427 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:35:46,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:35:46,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:35:46,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:35:46,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:35:46,428 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:35:46,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:35:46,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:35:46,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:35:46,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:46,429 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:46,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:46,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:46,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:46,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:46,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:46,463 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:46,487 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732415731261_ns4_test-17324156774073' hfile list 2024-11-24T02:35:46,569 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1708702679_0008_ff58720a-f7ac-4e13-9107-1f59d6f5d501/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:35:46,569 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1708702679_0008_ff58720a-f7ac-4e13-9107-1f59d6f5d501/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:35:47,605 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T02:35:47,607 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T02:35:47,616 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:47,616 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732415731261_ns4_test-17324156774073 finished. 2024-11-24T02:35:47,616 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732415722346_ns3:test-17324156774072 2024-11-24T02:35:47,616 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732415733423_ns3_test-17324156774072 to hdfs://localhost:40215/backupUT/backup_1732415722346/ns3/test-17324156774072/ 2024-11-24T02:35:47,616 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-24T02:35:47,642 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:40215, inputRoot=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:35:47,642 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_10852338_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40215/backupUT/backup_1732415722346/ns3/test-17324156774072, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40215/backupUT/backup_1732415722346/ns3/test-17324156774072/.hbase-snapshot/.tmp/snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:47,643 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-24T02:35:47,648 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_1732415733423_ns3_test-17324156774072 to hdfs://localhost:40215/backupUT/backup_1732415722346/ns3/test-17324156774072/.hbase-snapshot/.tmp/snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:47,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741946_1122 (size=86) 2024-11-24T02:35:47,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741947_1123 (size=443) 2024-11-24T02:35:48,062 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,063 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,063 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,921 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-9763897860597601282.jar 2024-11-24T02:35:48,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,922 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:35:48,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:35:48,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:35:48,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:35:48,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:35:48,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:35:48,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:35:48,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:35:48,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:35:48,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:35:48,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:35:48,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:35:48,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:48,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:48,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:48,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:48,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-24T02:35:48,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:48,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:35:48,956 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:35:48,979 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732415733423_ns3_test-17324156774072' hfile list 2024-11-24T02:35:49,063 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1309586929_0009_a6b9447b-ddcc-4962-806d-55a781b59f9c/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:35:49,063 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1309586929_0009_a6b9447b-ddcc-4962-806d-55a781b59f9c/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:35:50,100 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-24T02:35:50,101 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-24T02:35:50,110 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:50,110 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732415733423_ns3_test-17324156774072 finished. 2024-11-24T02:35:50,110 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-24T02:35:50,110 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732415677407 2024-11-24T02:35:50,110 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17324156774071 2024-11-24T02:35:50,110 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17324156774073 2024-11-24T02:35:50,110 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17324156774072 2024-11-24T02:35:50,119 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732415722346 2024-11-24T02:35:50,119 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-24T02:35:50,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741948_1124 (size=209) 2024-11-24T02:35:50,525 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:40215/backupUT/backup_1732415722346/.backup.manifest 2024-11-24T02:35:50,525 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732415722346 finished: type=FULL,tablelist=backup:system_bulk;test-1732415677407;ns2:test-17324156774071;ns4:test-17324156774073;ns3:test-17324156774072,targetRootDir=hdfs://localhost:40215/backupUT,startts=1732415724533,completets=1732415750119,bytescopied=0 2024-11-24T02:35:50,525 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-24T02:35:50,525 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:50,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732415724773_backup_system_bulk" type: DISABLED 2024-11-24T02:35:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732415724773_backup_system_bulk 2024-11-24T02:35:50,533 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732415724773_backup_system_bulk for backup backup_1732415722346 succeeded. 2024-11-24T02:35:50,533 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:50,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732415726932_default_test-1732415677407" type: DISABLED 2024-11-24T02:35:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732415726932_default_test-1732415677407 2024-11-24T02:35:50,540 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732415726932_default_test-1732415677407 for backup backup_1732415722346 succeeded. 2024-11-24T02:35:50,540 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:50,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732415729093_ns2_test-17324156774071" type: DISABLED 2024-11-24T02:35:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732415729093_ns2_test-17324156774071 2024-11-24T02:35:50,546 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732415729093_ns2_test-17324156774071 for backup backup_1732415722346 succeeded. 2024-11-24T02:35:50,546 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:50,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732415731261_ns4_test-17324156774073" type: DISABLED 2024-11-24T02:35:50,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732415731261_ns4_test-17324156774073 2024-11-24T02:35:50,549 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732415731261_ns4_test-17324156774073 for backup backup_1732415722346 succeeded. 2024-11-24T02:35:50,549 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:50,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732415733423_ns3_test-17324156774072" type: DISABLED 2024-11-24T02:35:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732415733423_ns3_test-17324156774072 2024-11-24T02:35:50,553 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732415733423_ns3_test-17324156774072 for backup backup_1732415722346 succeeded. 2024-11-24T02:35:50,554 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T02:35:50,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-24T02:35:50,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-24T02:35:50,561 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-24T02:35:50,563 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T02:35:50,567 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732415722346 completed. 2024-11-24T02:35:50,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,161 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:35:55,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,647 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-24T02:35:55,651 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732415755593 2024-11-24T02:35:55,653 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732415722346} 2024-11-24T02:35:55,653 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-24T02:35:55,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:35:55,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T02:35:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732415755750 (current time:1732415755750). 2024-11-24T02:35:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-24T02:35:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-24T02:35:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3501f948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:35:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:35:55,752 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:35:55,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:35:55,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:35:55,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b96ff3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:55,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:35:55,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:35:55,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:55,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:35:55,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a08ddc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:35:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:35:55,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:35:55,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:35:55,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:35:55,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423. 2024-11-24T02:35:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:35:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:35:55,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:35:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-24T02:35:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-24T02:35:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-24T02:35:55,761 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-24T02:35:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T02:35:55,763 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-24T02:35:55,765 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-24T02:35:55,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741949_1125 (size=52) 2024-11-24T02:35:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T02:35:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T02:35:56,174 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-24T02:35:56,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea}] 2024-11-24T02:35:56,177 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:56,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-24T02:35:56,331 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:35:56,331 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 6df51c7a7a1591e81897afbbac0da0ea 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-24T02:35:56,336 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/e7a6d46db1f94cfd8d505b6e62279e9b is 158, key is trslm:hdfs://localhost:40215/backupUT\x00ns2:test-17324156774071/meta:log-roll-map/1732415750114/Put/seqid=0 2024-11-24T02:35:56,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741950_1126 (size=6646) 2024-11-24T02:35:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T02:35:56,743 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/e7a6d46db1f94cfd8d505b6e62279e9b 2024-11-24T02:35:56,759 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/603448b939a54abaa2446ea7e889cc8e is 826, key is session:backup_1732415722346/session:context/1732415750562/Put/seqid=0 2024-11-24T02:35:56,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741951_1127 (size=5928) 2024-11-24T02:35:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T02:35:57,163 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/603448b939a54abaa2446ea7e889cc8e 2024-11-24T02:35:57,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/e7a6d46db1f94cfd8d505b6e62279e9b as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/e7a6d46db1f94cfd8d505b6e62279e9b 2024-11-24T02:35:57,180 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/e7a6d46db1f94cfd8d505b6e62279e9b, entries=12, sequenceid=42, filesize=6.5 K 2024-11-24T02:35:57,181 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/603448b939a54abaa2446ea7e889cc8e as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/603448b939a54abaa2446ea7e889cc8e 2024-11-24T02:35:57,186 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/603448b939a54abaa2446ea7e889cc8e, entries=2, sequenceid=42, filesize=5.8 K 2024-11-24T02:35:57,187 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 6df51c7a7a1591e81897afbbac0da0ea in 856ms, sequenceid=42, compaction requested=true 2024-11-24T02:35:57,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 6df51c7a7a1591e81897afbbac0da0ea: 2024-11-24T02:35:57,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. for snapshot_backup_system completed. 2024-11-24T02:35:57,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea.' region-info for snapshot=snapshot_backup_system 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/09439360a3384a58b0af1fd4ddfc50f5, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/6214c14cce634e038861bdb939ed2400, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/e7a6d46db1f94cfd8d505b6e62279e9b] hfiles 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/09439360a3384a58b0af1fd4ddfc50f5 for snapshot=snapshot_backup_system 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/6214c14cce634e038861bdb939ed2400 for snapshot=snapshot_backup_system 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/e7a6d46db1f94cfd8d505b6e62279e9b for snapshot=snapshot_backup_system 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/bc9a95a157c24f92aeeba767a22b2c63, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/ca5f74340e0d45e5b18e062fa6ed1c77, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/603448b939a54abaa2446ea7e889cc8e] hfiles 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/dc953a38304d4f09b0a4aa66f11b529d for snapshot=snapshot_backup_system 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/bc9a95a157c24f92aeeba767a22b2c63 for snapshot=snapshot_backup_system 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/ca5f74340e0d45e5b18e062fa6ed1c77 for snapshot=snapshot_backup_system 2024-11-24T02:35:57,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/603448b939a54abaa2446ea7e889cc8e for snapshot=snapshot_backup_system 2024-11-24T02:35:57,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741952_1128 (size=330) 2024-11-24T02:35:57,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:35:57,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-24T02:35:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-24T02:35:57,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:57,598 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:35:57,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T02:35:57,606 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-24T02:35:57,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 6df51c7a7a1591e81897afbbac0da0ea in 1.4270 sec 2024-11-24T02:35:57,607 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-24T02:35:57,608 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-24T02:35:57,608 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-24T02:35:57,609 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-24T02:35:57,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741953_1129 (size=967) 2024-11-24T02:35:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T02:35:58,020 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-24T02:35:58,031 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-24T02:35:58,032 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.hbase-snapshot/snapshot_backup_system 2024-11-24T02:35:58,033 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-24T02:35:58,033 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-24T02:35:58,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2740 sec 2024-11-24T02:35:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T02:35:59,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-24T02:35:59,933 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732415755593 started at 1732415759933. 2024-11-24T02:35:59,939 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732415755593 has been started. 2024-11-24T02:35:59,948 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, test-1732415677407, ns2:test-17324156774071, ns3:test-17324156774072, ns4:test-17324156774073] 2024-11-24T02:35:59,952 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732415712917for backupID backup_1732415755593 2024-11-24T02:35:59,953 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-24T02:35:59,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-24T02:35:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-24T02:35:59,954 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-24T02:35:59,954 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-24T02:35:59,954 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-24T02:35:59,954 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-24T02:35:59,955 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T02:35:59,955 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T02:36:00,013 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T02:36:00,014 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T02:36:00,014 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:36:00,014 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,014 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-24T02:36:00,014 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-24T02:36:00,015 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-24T02:36:00,015 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-24T02:36:00,015 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-24T02:36:00,015 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-24T02:36:00,016 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-24T02:36:00,016 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-24T02:36:00,016 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-24T02:36:00,016 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-24T02:36:00,016 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-24T02:36:00,016 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-24T02:36:00,016 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'a67e7d17e593,38807,1732415667927' joining acquired barrier for procedure (rolllog) in zk 2024-11-24T02:36:00,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,029 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-24T02:36:00,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:36:00,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:36:00,029 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-24T02:36:00,029 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-24T02:36:00,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:36:00,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:36:00,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:36:00,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:36:00,032 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'a67e7d17e593,38807,1732415667927' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-24T02:36:00,032 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@357880b7[Count = 0] remaining members to acquire global barrier 2024-11-24T02:36:00,032 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-24T02:36:00,033 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-24T02:36:00,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:36:00,043 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-24T02:36:00,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-24T02:36:00,043 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-24T02:36:00,043 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-24T02:36:00,043 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,043 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-24T02:36:00,043 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,043 INFO [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732415724643 highest: 1732415724679 on a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,043 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0:(num 1732415724643) roll requested 2024-11-24T02:36:00,057 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415760044, exclude list is [], retry=0 2024-11-24T02:36:00,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:36:00,062 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415760044 2024-11-24T02:36:00,062 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:36:00,062 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 is not closed yet, will try archiving it next time 2024-11-24T02:36:00,062 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.meta:.meta(num 1732415724663) roll requested 2024-11-24T02:36:00,062 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415693359 2024-11-24T02:36:00,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741911_1087 (size=653) 2024-11-24T02:36:00,063 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 not finished, retry = 0 2024-11-24T02:36:00,074 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415760062.meta, exclude list is [], retry=0 2024-11-24T02:36:00,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:36:00,077 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415724663.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415760062.meta 2024-11-24T02:36:00,077 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:36:00,077 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415724663.meta is not closed yet, will try archiving it next time 2024-11-24T02:36:00,077 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1:(num 1732415724679) roll requested 2024-11-24T02:36:00,079 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415724663.meta not finished, retry = 0 2024-11-24T02:36:00,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741912_1088 (size=93) 2024-11-24T02:36:00,089 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415760077, exclude list is [], retry=0 2024-11-24T02:36:00,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34653,DS-4508af20-af3c-4cc1-a699-29e9b8c67228,DISK] 2024-11-24T02:36:00,093 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415760077 2024-11-24T02:36:00,093 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43173:43173)] 2024-11-24T02:36:00,093 DEBUG [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679 is not closed yet, will try archiving it next time 2024-11-24T02:36:00,093 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415712917 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415712917 2024-11-24T02:36:00,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741913_1089 (size=4710) 2024-11-24T02:36:00,104 DEBUG [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-24T02:36:00,104 INFO [rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732415760044 on a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:00,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:00,125 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-24T02:36:00,125 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'a67e7d17e593,38807,1732415667927' in zk 2024-11-24T02:36:00,142 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-24T02:36:00,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,142 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T02:36:00,143 DEBUG [member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-24T02:36:00,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:36:00,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:36:00,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:36:00,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:36:00,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:36:00,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:36:00,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:36:00,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'a67e7d17e593,38807,1732415667927': 2024-11-24T02:36:00,148 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'a67e7d17e593,38807,1732415667927' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-24T02:36:00,148 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-24T02:36:00,148 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-24T02:36:00,149 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-24T02:36:00,149 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-24T02:36:00,149 INFO [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-24T02:36:00,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:36:00,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:36:00,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T02:36:00,159 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-24T02:36:00,159 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-24T02:36:00,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-24T02:36:00,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-24T02:36:00,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T02:36:00,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-24T02:36:00,161 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T02:36:00,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:36:00,161 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:36:00,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-24T02:36:00,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-24T02:36:00,164 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:36:00,165 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,165 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 2024-11-24T02:36:00,165 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-24T02:36:00,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-24T02:36:00,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-24T02:36:00,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T02:36:00,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-24T02:36:00,179 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-24T02:36:00,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:00,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-24T02:36:00,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:36:00,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-24T02:36:00,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-24T02:36:00,180 DEBUG [(a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-24T02:36:00,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-24T02:36:00,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-24T02:36:00,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-24T02:36:00,180 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415724663.meta to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.meta.1732415724663.meta 2024-11-24T02:36:00,180 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-24T02:36:00,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-24T02:36:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-24T02:36:00,197 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {a67e7d17e593:38807=1732415712917} newestTimestamps: {a67e7d17e593:38807=1732415724679} 2024-11-24T02:36:00,200 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: a67e7d17e593%2C38807%2C1732415667927.meta.1732415672505.meta 2024-11-24T02:36:00,200 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: a67e7d17e593%2C38807%2C1732415667927.meta.1732415712901.meta 2024-11-24T02:36:00,200 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: a67e7d17e593%2C38807%2C1732415667927.meta.1732415760062.meta 2024-11-24T02:36:00,200 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415760044 2024-11-24T02:36:00,200 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679 2024-11-24T02:36:00,200 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415760077 2024-11-24T02:36:00,202 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.meta.1732415693377.meta 2024-11-24T02:36:00,202 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.meta.1732415724663.meta 2024-11-24T02:36:00,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741957_1133 (size=640) 2024-11-24T02:36:00,615 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:40215/backupUT/backup_1732415755593/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:00,616 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T02:36:00,622 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-24T02:36:00,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741958_1134 (size=45) 2024-11-24T02:36:01,030 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-24T02:36:01,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741959_1135 (size=950) 2024-11-24T02:36:01,447 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732415677407 target: hdfs://localhost:40215/backupUT/backup_1732415755593/default/test-1732415677407 descriptor: 'test-1732415677407', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:01,447 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T02:36:01,450 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732415677407 2024-11-24T02:36:01,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741960_1136 (size=53) 2024-11-24T02:36:01,855 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732415677407 2024-11-24T02:36:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741961_1137 (size=364) 2024-11-24T02:36:02,267 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17324156774071 target: hdfs://localhost:40215/backupUT/backup_1732415755593/ns2/test-17324156774071 descriptor: 'ns2:test-17324156774071', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:02,267 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T02:36:02,273 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17324156774071 2024-11-24T02:36:02,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741962_1138 (size=50) 2024-11-24T02:36:02,680 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17324156774071 2024-11-24T02:36:02,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741963_1139 (size=390) 2024-11-24T02:36:03,091 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17324156774073 target: hdfs://localhost:40215/backupUT/backup_1732415755593/ns4/test-17324156774073 descriptor: 'ns4:test-17324156774073', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:03,091 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T02:36:03,093 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17324156774073 2024-11-24T02:36:03,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741964_1140 (size=50) 2024-11-24T02:36:03,500 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17324156774073 2024-11-24T02:36:03,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741965_1141 (size=390) 2024-11-24T02:36:03,920 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17324156774072 target: hdfs://localhost:40215/backupUT/backup_1732415755593/ns3/test-17324156774072 descriptor: 'ns3:test-17324156774072', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:03,920 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-24T02:36:03,922 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17324156774072 2024-11-24T02:36:03,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741966_1142 (size=50) 2024-11-24T02:36:04,329 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17324156774072 2024-11-24T02:36:04,349 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:40215/backupUT/.tmp/backup_1732415755593 from hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679;hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 2024-11-24T02:36:04,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c8b2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:04,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:04,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:04,351 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:04,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:04,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:04,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1923e428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:04,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:04,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:04,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:04,353 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:04,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0528a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:04,354 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:04,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:04,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f522a25 2024-11-24T02:36:04,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:04,355 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:04,358 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T02:36:04,358 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-24T02:36:04,358 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:04,359 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:04,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:04,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:04,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:04,362 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-24T02:36:04,362 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732415677407 2024-11-24T02:36:04,362 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732415677407,, stopping at row=test-1732415677407 ,, for max=2147483647 with caching=100 2024-11-24T02:36:04,364 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732415677407: test-1732415677407; 2024-11-24T02:36:04,364 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17324156774071 2024-11-24T02:36:04,364 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17324156774071,, stopping at row=ns2:test-17324156774071 ,, for max=2147483647 with caching=100 2024-11-24T02:36:04,366 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17324156774071: ns2:test-17324156774071; 2024-11-24T02:36:04,366 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17324156774072 2024-11-24T02:36:04,366 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17324156774072,, stopping at row=ns3:test-17324156774072 ,, for max=2147483647 with caching=100 2024-11-24T02:36:04,368 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17324156774072: ns3:test-17324156774072; 2024-11-24T02:36:04,368 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17324156774073 2024-11-24T02:36:04,368 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17324156774073,, stopping at row=ns4:test-17324156774073 ,, for max=2147483647 with caching=100 2024-11-24T02:36:04,370 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17324156774073: ns4:test-17324156774073; 2024-11-24T02:36:04,370 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-24T02:36:04,370 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/partitions_d1906617-157f-429d-8bd1-417387b2a403 2024-11-24T02:36:04,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741967_1143 (size=324) 2024-11-24T02:36:04,780 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:04,781 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:04,781 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,641 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-5235656757498815097.jar 2024-11-24T02:36:05,641 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,642 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,643 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,643 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,643 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:05,643 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:36:05,643 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:36:05,644 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:36:05,644 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:36:05,644 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:36:05,644 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:36:05,644 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:36:05,644 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:36:05,645 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:36:05,645 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:36:05,645 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,645 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,645 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,645 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,646 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,646 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:05,646 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:36:05,646 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,default:test-1732415677407,ns2:test-17324156774071,ns3:test-17324156774072,ns4:test-17324156774073 2024-11-24T02:36:05,646 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:05,646 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:05,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:05,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:05,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:05,647 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:06,499 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-11737473659482082008.jar 2024-11-24T02:36:06,507 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:36:06,530 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679 for WAL files 2024-11-24T02:36:06,531 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679 2024-11-24T02:36:06,531 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 for WAL files 2024-11-24T02:36:06,532 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 2024-11-24T02:36:06,604 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local817378579_0010_0c7d57e4-3de5-4917-964d-0c47af96a8fe/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:36:06,604 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local817378579_0010_0c7d57e4-3de5-4917-964d-0c47af96a8fe/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:36:06,653 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679 for hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415724679 (-9223372036854775808:9223372036854775807) length:4710 2024-11-24T02:36:06,657 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-24T02:36:06,657 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-24T02:36:06,688 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 for hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415724643 (-9223372036854775808:9223372036854775807) length:653 2024-11-24T02:36:06,691 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-24T02:36:06,691 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-24T02:36:06,766 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c837cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:06,767 DEBUG [pool-260-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:06,767 DEBUG [pool-260-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:06,768 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:06,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:06,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:06,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3dc2a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:06,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:06,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:06,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:06,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:06,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1585093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:06,770 DEBUG [pool-260-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:06,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:06,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:06,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:06,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732415677407', row='row-test-1732415677407-2-0', locateType=CURRENT is [region=test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., hostname=a67e7d17e593,38807,1732415667927, seqNum=338] 2024-11-24T02:36:06,774 INFO [pool-260-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-260-thread-1. 2024-11-24T02:36:06,774 DEBUG [pool-260-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:06,774 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:06,774 DEBUG [pool-260-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:06,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:06,774 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732415677407-2-0] 2024-11-24T02:36:06,774 DEBUG [pool-260-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:06,778 DEBUG [pool-260-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/backupUT/.tmp/backup_1732415755593/_temporary/0/_temporary/attempt_local817378579_0010_r_000004_0/default/test-1732415677407/f2/a39a2599bbd44757a26c9170f1156673 is 58, key is row-test-1732415677407-2-0/f2:q1/1732415750583/Put/seqid=0 2024-11-24T02:36:06,779 WARN [Thread-1517 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:06,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741968_1144 (size=5366) 2024-11-24T02:36:07,177 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region a9fcfc65d96498f86ae4291b18ffd315, had cached 0 bytes from a total of 38453 2024-11-24T02:36:07,651 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:40215/backupUT 2024-11-24T02:36:07,651 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732415755593 2024-11-24T02:36:07,651 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-24T02:36:07,663 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:40215/backupUT/.tmp/backup_1732415755593, hdfs://localhost:40215/backupUT] 2024-11-24T02:36:07,713 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:36:07,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741969_1145 (size=5366) 2024-11-24T02:36:08,327 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-24T02:36:08,330 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732415755593 2024-11-24T02:36:08,330 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 5366 bytes copied." 2024-11-24T02:36:08,330 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local709037051_0011 completed: true true 2024-11-24T02:36:08,331 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=6669335 FILE: Number of bytes written=17153782 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=410445 HDFS: Number of bytes written=2548319 HDFS: Number of read operations=1165 HDFS: Number of large read operations=0 HDFS: Number of write operations=424 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=338 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=1482 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=5366 Bytes Copied=5366 Bytes Expected=5366 Files Copied=2 DIR_COPY=4 2024-11-24T02:36:08,331 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:40215/backupUT/.tmp/backup_1732415755593 to hdfs://localhost:40215/backupUT finished. 2024-11-24T02:36:08,351 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732415755593 2024-11-24T02:36:08,355 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732415722346} 2024-11-24T02:36:08,355 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-24T02:36:08,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741970_1146 (size=714) 2024-11-24T02:36:08,762 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:40215/backupUT/backup_1732415755593/.backup.manifest 2024-11-24T02:36:08,762 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732415755593 finished: type=INCREMENTAL,tablelist=backup:system_bulk;test-1732415677407;ns2:test-17324156774071;ns4:test-17324156774073;ns3:test-17324156774072,targetRootDir=hdfs://localhost:40215/backupUT,startts=1732415759933,completets=1732415768351,bytescopied=0 2024-11-24T02:36:08,762 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-24T02:36:08,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-24T02:36:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-24T02:36:08,776 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-24T02:36:08,780 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-24T02:36:08,784 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732415755593 completed. 2024-11-24T02:36:08,784 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-24T02:36:08,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:08,802 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(191): Restoring full backup_1732415687521 2024-11-24T02:36:08,805 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table table1 does not exist. It will be created during restore process 2024-11-24T02:36:08,806 INFO [Time-limited test {}] impl.RestoreTablesClient(100): HBase table ns2:table2 does not exist. It will be created during restore process 2024-11-24T02:36:08,807 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:40215/backupUT/backup_1732415687521/.backup.manifest 2024-11-24T02:36:08,809 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732415687521/.backup.manifest 2024-11-24T02:36:08,811 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:40215/backupUT/backup_1732415687521/.backup.manifest 2024-11-24T02:36:08,813 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732415687521/.backup.manifest 2024-11-24T02:36:08,813 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1732415677407' to 'table1' from full backup image hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407 2024-11-24T02:36:08,821 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'table1' 2024-11-24T02:36:08,821 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:36:08,822 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f in region [hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315] 2024-11-24T02:36:08,827 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 first=row0 last=row98 2024-11-24T02:36:08,827 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3 in region [hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315] 2024-11-24T02:36:08,830 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 first=row-test-1732415677407-3-0 last=row-test-1732415677407-3-5 2024-11-24T02:36:08,830 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob in region [hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315] 2024-11-24T02:36:08,833 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c first=row-test-1732415677407-3-0 last=row-test-1732415677407-3-5 2024-11-24T02:36:08,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-24T02:36:08,840 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:36:08,840 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:08,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 62 2024-11-24T02:36:08,841 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:36:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:36:08,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741971_1147 (size=938) 2024-11-24T02:36:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:36:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:36:09,253 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c8065a920e0f1af7868a104494e4732d, NAME => 'table1,,1732415768837.c8065a920e0f1af7868a104494e4732d.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:36:09,254 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1523f0656aca8ceaf15f6bcfbf77d271, NAME => 'table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:36:09,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741972_1148 (size=45) 2024-11-24T02:36:09,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741973_1149 (size=45) 2024-11-24T02:36:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:36:09,667 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1732415768837.c8065a920e0f1af7868a104494e4732d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:09,667 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing c8065a920e0f1af7868a104494e4732d, disabling compactions & flushes 2024-11-24T02:36:09,667 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:09,667 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:09,667 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:09,667 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. after waiting 0 ms 2024-11-24T02:36:09,668 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:09,668 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing 1523f0656aca8ceaf15f6bcfbf77d271, disabling compactions & flushes 2024-11-24T02:36:09,668 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:09,668 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:09,668 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:09,668 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for c8065a920e0f1af7868a104494e4732d: Waiting for close lock at 1732415769667Disabling compacts and flushes for region at 1732415769667Disabling writes for close at 1732415769668 (+1 ms)Writing region close event to WAL at 1732415769668Closed at 1732415769668 2024-11-24T02:36:09,668 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. after waiting 0 ms 2024-11-24T02:36:09,668 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:09,668 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:09,668 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1523f0656aca8ceaf15f6bcfbf77d271: Waiting for close lock at 1732415769668Disabling compacts and flushes for region at 1732415769668Disabling writes for close at 1732415769668Writing region close event to WAL at 1732415769668Closed at 1732415769668 2024-11-24T02:36:09,671 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:36:09,672 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1732415768837.c8065a920e0f1af7868a104494e4732d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732415769671"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415769671"}]},"ts":"1732415769671"} 2024-11-24T02:36:09,672 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732415769671"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415769671"}]},"ts":"1732415769671"} 2024-11-24T02:36:09,677 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-11-24T02:36:09,678 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:36:09,678 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415769678"}]},"ts":"1732415769678"} 2024-11-24T02:36:09,680 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-24T02:36:09,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, ASSIGN}, {pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, ASSIGN}] 2024-11-24T02:36:09,682 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, ASSIGN 2024-11-24T02:36:09,682 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, ASSIGN 2024-11-24T02:36:09,682 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:36:09,682 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:36:09,834 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1523f0656aca8ceaf15f6bcfbf77d271, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:09,834 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=c8065a920e0f1af7868a104494e4732d, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:09,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, ASSIGN because future has completed 2024-11-24T02:36:09,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8065a920e0f1af7868a104494e4732d, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:09,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, ASSIGN because future has completed 2024-11-24T02:36:09,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1523f0656aca8ceaf15f6bcfbf77d271, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:36:10,002 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:10,003 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => c8065a920e0f1af7868a104494e4732d, NAME => 'table1,,1732415768837.c8065a920e0f1af7868a104494e4732d.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-24T02:36:10,003 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:36:10,003 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,003 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated table1,,1732415768837.c8065a920e0f1af7868a104494e4732d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:10,004 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,004 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,005 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,007 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8065a920e0f1af7868a104494e4732d columnFamilyName f 2024-11-24T02:36:10,007 DEBUG [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:10,008 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] regionserver.HStore(327): Store=c8065a920e0f1af7868a104494e4732d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:10,008 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,009 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8065a920e0f1af7868a104494e4732d columnFamilyName f3 2024-11-24T02:36:10,010 DEBUG [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:10,010 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] regionserver.HStore(327): Store=c8065a920e0f1af7868a104494e4732d/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:10,010 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,011 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8065a920e0f1af7868a104494e4732d columnFamilyName mob 2024-11-24T02:36:10,012 DEBUG [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:10,012 INFO [StoreOpener-c8065a920e0f1af7868a104494e4732d-1 {}] regionserver.HStore(327): Store=c8065a920e0f1af7868a104494e4732d/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:10,012 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,013 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,013 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,014 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,014 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,015 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T02:36:10,016 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,018 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:36:10,018 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened c8065a920e0f1af7868a104494e4732d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11022727520, jitterRate=0.026571497321128845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T02:36:10,018 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:10,019 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for c8065a920e0f1af7868a104494e4732d: Running coprocessor pre-open hook at 1732415770004Writing region info on filesystem at 1732415770004Initializing all the Stores at 1732415770005 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415770005Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415770005Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415770005Cleaning up temporary data from old regions at 1732415770014 (+9 ms)Running coprocessor post-open hooks at 1732415770018 (+4 ms)Region opened successfully at 1732415770019 (+1 ms) 2024-11-24T02:36:10,020 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., pid=65, masterSystemTime=1732415769993 2024-11-24T02:36:10,022 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:10,022 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:10,022 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] handler.AssignRegionHandler(132): Open table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:10,022 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7752): Opening region: {ENCODED => 1523f0656aca8ceaf15f6bcfbf77d271, NAME => 'table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-24T02:36:10,022 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:36:10,022 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=c8065a920e0f1af7868a104494e4732d, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:10,022 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,022 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(898): Instantiated table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:10,022 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7794): checking encryption for 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,023 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(7797): checking classloading for 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8065a920e0f1af7868a104494e4732d, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:10,024 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,026 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523f0656aca8ceaf15f6bcfbf77d271 columnFamilyName f 2024-11-24T02:36:10,026 DEBUG [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:10,026 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] regionserver.HStore(327): Store=1523f0656aca8ceaf15f6bcfbf77d271/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:10,026 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=63 2024-11-24T02:36:10,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure c8065a920e0f1af7868a104494e4732d, server=a67e7d17e593,38807,1732415667927 in 186 msec 2024-11-24T02:36:10,029 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523f0656aca8ceaf15f6bcfbf77d271 columnFamilyName f3 2024-11-24T02:36:10,029 DEBUG [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:10,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, ASSIGN in 347 msec 2024-11-24T02:36:10,029 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] regionserver.HStore(327): Store=1523f0656aca8ceaf15f6bcfbf77d271/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:10,029 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,031 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1523f0656aca8ceaf15f6bcfbf77d271 columnFamilyName mob 2024-11-24T02:36:10,031 DEBUG [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:10,032 INFO [StoreOpener-1523f0656aca8ceaf15f6bcfbf77d271-1 {}] regionserver.HStore(327): Store=1523f0656aca8ceaf15f6bcfbf77d271/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:10,032 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1038): replaying wal for 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,033 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,033 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,034 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1048): stopping wal replay for 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,034 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1060): Cleaning up temporary data for 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,034 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T02:36:10,035 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1093): writing seq id for 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,037 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:36:10,037 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1114): Opened 1523f0656aca8ceaf15f6bcfbf77d271; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11310051840, jitterRate=0.05333065986633301}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T02:36:10,037 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:10,037 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegion(1006): Region open journal for 1523f0656aca8ceaf15f6bcfbf77d271: Running coprocessor pre-open hook at 1732415770023Writing region info on filesystem at 1732415770023Initializing all the Stores at 1732415770024 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415770024Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415770024Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415770024Cleaning up temporary data from old regions at 1732415770034 (+10 ms)Running coprocessor post-open hooks at 1732415770037 (+3 ms)Region opened successfully at 1732415770037 2024-11-24T02:36:10,038 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271., pid=66, masterSystemTime=1732415769993 2024-11-24T02:36:10,039 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:10,039 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=66}] handler.AssignRegionHandler(153): Opened table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:10,040 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1523f0656aca8ceaf15f6bcfbf77d271, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:10,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1523f0656aca8ceaf15f6bcfbf77d271, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:10,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=64 2024-11-24T02:36:10,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 1523f0656aca8ceaf15f6bcfbf77d271, server=a67e7d17e593,38807,1732415667927 in 201 msec 2024-11-24T02:36:10,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=62 2024-11-24T02:36:10,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=62, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, ASSIGN in 364 msec 2024-11-24T02:36:10,046 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:36:10,046 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415770046"}]},"ts":"1732415770046"} 2024-11-24T02:36:10,047 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-24T02:36:10,048 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=62, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:36:10,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 1.2110 sec 2024-11-24T02:36:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:36:10,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-24T02:36:10,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:11,002 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 from hbase tables test-1732415677407 to tables table1 2024-11-24T02:36:11,002 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1732415677407 into table1 2024-11-24T02:36:11,006 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002 from hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:36:11,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5419b46e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:11,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:11,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:11,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:11,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:11,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:11,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a41eea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:11,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:11,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:11,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:11,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:11,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f3c7b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:11,011 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:11,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:11,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54e4dabf 2024-11-24T02:36:11,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:11,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:11,013 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T02:36:11,013 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-24T02:36:11,014 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:11,014 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:11,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:11,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:11,015 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 50af115aeed7c572d01a2d7d6548f037, had cached 0 bytes from a total of 13547 2024-11-24T02:36:11,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:11,017 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-24T02:36:11,017 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-24T02:36:11,017 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-24T02:36:11,017 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/partitions_77d3af55-8598-4747-8e62-65487f4f84dc 2024-11-24T02:36:11,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741974_1150 (size=177) 2024-11-24T02:36:11,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:11,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:11,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-610028744078926853.jar 2024-11-24T02:36:12,287 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,287 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,287 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,287 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,287 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,288 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,288 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,288 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,288 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,288 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:12,288 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:36:12,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:36:12,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:36:12,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:36:12,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:36:12,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:36:12,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:36:12,289 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:36:12,290 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:36:12,290 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:36:12,291 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,292 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,292 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,292 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,292 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,292 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:12,292 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:36:12,292 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-24T02:36:12,293 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:12,293 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:12,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:12,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:12,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:12,293 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T02:36:12,293 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:12,308 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:36:12,323 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T02:36:12,401 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local755474873_0012_06c5f3e3-5608-45e2-998a-91d150552d38/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:36:12,401 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local755474873_0012_06c5f3e3-5608-45e2-998a-91d150552d38/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:36:12,447 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 2024-11-24T02:36:12,450 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:12,484 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c 2024-11-24T02:36:12,487 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:12,506 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f3/dc0694660d6d4041923141ad81e67545 2024-11-24T02:36:12,509 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:12,535 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e1947d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:12,535 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:12,535 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:12,536 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:12,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:12,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:12,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b763913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:12,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:12,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:12,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:12,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:12,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fec5155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:12,538 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:12,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:12,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:12,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:12,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732415677407-3-0', locateType=CURRENT is [region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:12,542 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-24T02:36:12,542 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:12,542 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:12,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:12,542 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:12,542 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732415677407-3-0] 2024-11-24T02:36:12,542 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:12,547 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d11ec0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:12,547 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:12,547 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:12,548 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:12,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:12,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:12,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5583d490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:12,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:12,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:12,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:12,549 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:12,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775b687b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:12,550 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:12,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:12,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:12,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:12,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732415677407-3-0', locateType=CURRENT is [region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:12,554 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-24T02:36:12,554 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:12,554 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:12,554 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:12,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:12,554 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732415677407-3-0] 2024-11-24T02:36:12,554 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:12,558 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/_temporary/0/_temporary/attempt_local755474873_0012_r_000000_0/f3/6dbcdbc1fbef4168b305091d84aafc47 is 58, key is row-test-1732415677407-3-0/f3:q1/1732415687484/Put/seqid=0 2024-11-24T02:36:12,560 WARN [Thread-1636 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:12,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741975_1151 (size=5529) 2024-11-24T02:36:12,966 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/_temporary/0/_temporary/attempt_local755474873_0012_r_000000_0/mob/55f94161c39542babe8a841774e07485 is 59, key is row-test-1732415677407-3-0/mob:q1/1732415687503/Put/seqid=0 2024-11-24T02:36:12,971 WARN [Thread-1638 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:12,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741976_1152 (size=5538) 2024-11-24T02:36:13,409 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f936c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:13,409 DEBUG [pool-267-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:13,409 DEBUG [pool-267-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:13,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:13,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:13,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:13,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9adacb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:13,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:13,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:13,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:13,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:13,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cd2f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:13,412 DEBUG [pool-267-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:13,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:13,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:13,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:13,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:13,417 INFO [pool-267-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-267-thread-1. 2024-11-24T02:36:13,417 DEBUG [pool-267-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:13,417 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:13,417 DEBUG [pool-267-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:13,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:13,417 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-24T02:36:13,417 DEBUG [pool-267-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:13,426 DEBUG [pool-267-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/_temporary/0/_temporary/attempt_local755474873_0012_r_000001_0/f/89588421888f46c3937ff1af06c2e2b8 is 37, key is row10/f:q1/1732415680115/Put/seqid=0 2024-11-24T02:36:13,427 WARN [Thread-1653 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:13,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741977_1153 (size=8741) 2024-11-24T02:36:13,550 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-24T02:36:14,439 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002 2024-11-24T02:36:14,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db230e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:14,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:14,440 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:14,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:14,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:14,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284bc1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:14,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:14,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,442 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:14,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f698fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,443 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:14,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:14,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:14,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:14,448 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/_SUCCESS 2024-11-24T02:36:14,457 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:14,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:14,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ac7ae0b 2024-11-24T02:36:14,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:14,459 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:14,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:14,469 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,478 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/f/89588421888f46c3937ff1af06c2e2b8 first=Optional[row0] last=Optional[row98] 2024-11-24T02:36:14,482 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/f3/6dbcdbc1fbef4168b305091d84aafc47 first=Optional[row-test-1732415677407-3-0] last=Optional[row-test-1732415677407-3-5] 2024-11-24T02:36:14,485 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/mob/55f94161c39542babe8a841774e07485 first=Optional[row-test-1732415677407-3-0] last=Optional[row-test-1732415677407-3-5] 2024-11-24T02:36:14,500 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/f/89588421888f46c3937ff1af06c2e2b8 for inclusion in 1523f0656aca8ceaf15f6bcfbf77d271/f 2024-11-24T02:36:14,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/f3/6dbcdbc1fbef4168b305091d84aafc47 for inclusion in c8065a920e0f1af7868a104494e4732d/f3 2024-11-24T02:36:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732415677407-3-0 last=row-test-1732415677407-3-5 2024-11-24T02:36:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-24T02:36:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-11-24T02:36:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T02:36:14,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/mob/55f94161c39542babe8a841774e07485 for inclusion in c8065a920e0f1af7868a104494e4732d/mob 2024-11-24T02:36:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HRegion(2603): Flush status journal for 1523f0656aca8ceaf15f6bcfbf77d271: 2024-11-24T02:36:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/f/89588421888f46c3937ff1af06c2e2b8 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f/89588421888f46c3937ff1af06c2e2b8 2024-11-24T02:36:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732415677407-3-0 last=row-test-1732415677407-3-5 2024-11-24T02:36:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T02:36:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(2603): Flush status journal for c8065a920e0f1af7868a104494e4732d: 2024-11-24T02:36:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f/89588421888f46c3937ff1af06c2e2b8 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/f/295bc44d42b74e30a82fec1b9e01d521_SeqId_4_ 2024-11-24T02:36:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/f3/6dbcdbc1fbef4168b305091d84aafc47 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f3/6dbcdbc1fbef4168b305091d84aafc47 2024-11-24T02:36:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15fbeef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:14,512 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:14,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:14,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:14,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f86cc0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:14,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:14,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f3/6dbcdbc1fbef4168b305091d84aafc47 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/f3/57f7d99b580748429edb5e89094f9f1c_SeqId_4_ 2024-11-24T02:36:14,514 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415771002/mob/55f94161c39542babe8a841774e07485 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/mob/55f94161c39542babe8a841774e07485 2024-11-24T02:36:14,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683b08cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:14,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:14,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43e324cf 2024-11-24T02:36:14,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/mob/55f94161c39542babe8a841774e07485 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/mob/39cd543bd788428ab64249d09a124c2c_SeqId_4_ 2024-11-24T02:36:14,516 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d61245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:14,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:14,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:14,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:14,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd4d7f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:14,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:14,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:14,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:14,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:14,519 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:14,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724a69a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:14,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:14,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@74098e04 2024-11-24T02:36:14,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:14,522 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:14,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:14,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:14,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:14,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:14,540 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807. 2024-11-24T02:36:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:14,541 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f/89588421888f46c3937ff1af06c2e2b8 into 1523f0656aca8ceaf15f6bcfbf77d271/f as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/f/295bc44d42b74e30a82fec1b9e01d521_SeqId_4_ - updating store file list. 2024-11-24T02:36:14,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:14,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/f/295bc44d42b74e30a82fec1b9e01d521_SeqId_4_ into 1523f0656aca8ceaf15f6bcfbf77d271/f 2024-11-24T02:36:14,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f/89588421888f46c3937ff1af06c2e2b8 into 1523f0656aca8ceaf15f6bcfbf77d271/f (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/f/295bc44d42b74e30a82fec1b9e01d521_SeqId_4_) 2024-11-24T02:36:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f/89588421888f46c3937ff1af06c2e2b8 2024-11-24T02:36:14,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63628a94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:14,552 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:14,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:14,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:14,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6355c486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:14,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:14,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:14,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf75654, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df3bfd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:14,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:14,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a592275 2024-11-24T02:36:14,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:14,556 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:14,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:14,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:14,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a09f847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:14,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:14,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:14,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,557 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:14,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c873d02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:14,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:14,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:14,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:14,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b2ebdf 2024-11-24T02:36:14,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:14,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:14,560 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T02:36:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:14,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:14,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:14,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T02:36:14,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:14,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:14,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:14,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f3/6dbcdbc1fbef4168b305091d84aafc47 into c8065a920e0f1af7868a104494e4732d/f3 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/f3/57f7d99b580748429edb5e89094f9f1c_SeqId_4_ - updating store file list. 2024-11-24T02:36:14,581 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807. 2024-11-24T02:36:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:14,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/f3/57f7d99b580748429edb5e89094f9f1c_SeqId_4_ into c8065a920e0f1af7868a104494e4732d/f3 2024-11-24T02:36:14,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f3/6dbcdbc1fbef4168b305091d84aafc47 into c8065a920e0f1af7868a104494e4732d/f3 (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/f3/57f7d99b580748429edb5e89094f9f1c_SeqId_4_) 2024-11-24T02:36:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/f3/6dbcdbc1fbef4168b305091d84aafc47 2024-11-24T02:36:14,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/mob/55f94161c39542babe8a841774e07485 into c8065a920e0f1af7868a104494e4732d/mob as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/mob/39cd543bd788428ab64249d09a124c2c_SeqId_4_ - updating store file list. 2024-11-24T02:36:14,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/mob/39cd543bd788428ab64249d09a124c2c_SeqId_4_ into c8065a920e0f1af7868a104494e4732d/mob 2024-11-24T02:36:14,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/mob/55f94161c39542babe8a841774e07485 into c8065a920e0f1af7868a104494e4732d/mob (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/mob/39cd543bd788428ab64249d09a124c2c_SeqId_4_) 2024-11-24T02:36:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__86h7cbi3cd21jgacdo45fajstssoodqdsa9e62cedo63udqc6dkm9jlkhc9bfe2f/mob/55f94161c39542babe8a841774e07485 2024-11-24T02:36:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@edefb2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:14,593 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:14,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:14,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:14,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ed67cd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:14,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:14,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,594 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:14,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499ddb4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:14,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:14,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b9143d5 2024-11-24T02:36:14,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:14,597 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T02:36:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:14,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:14,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:14,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T02:36:14,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:14,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:14,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:14,616 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:14,616 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:14,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:14,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:14,617 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T02:36:14,617 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-24T02:36:14,617 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1732415687521 hdfs://localhost:40215/backupUT/backup_1732415687521/default/test-1732415677407/ 2024-11-24T02:36:14,618 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:40215/backupUT/backup_1732415687521/.backup.manifest 2024-11-24T02:36:14,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 , the old value is region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:14,618 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T02:36:14,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:14,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 from cache 2024-11-24T02:36:14,620 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732415687521/.backup.manifest 2024-11-24T02:36:14,620 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17324156774071' to 'ns2:table2' from full backup image hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071 2024-11-24T02:36:14,628 INFO [Time-limited test {}] util.RestoreTool(491): Creating target table 'ns2:table2' 2024-11-24T02:36:14,628 DEBUG [Time-limited test {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:36:14,629 DEBUG [Time-limited test {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f in region [hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037] 2024-11-24T02:36:14,632 DEBUG [Time-limited test {}] util.RestoreTool(446): Trying to figure out region boundaries hfile=hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 first=row0 last=row98 2024-11-24T02:36:14,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:table2 2024-11-24T02:36:14,635 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:36:14,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 67 2024-11-24T02:36:14,635 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:36:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T02:36:14,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741978_1154 (size=351) 2024-11-24T02:36:14,734 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 34 more 2024-11-24T02:36:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T02:36:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T02:36:15,042 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7253aee85dc96326b8dc14f1506394b7, NAME => 'ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:36:15,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741979_1155 (size=37) 2024-11-24T02:36:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T02:36:15,449 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:15,449 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 7253aee85dc96326b8dc14f1506394b7, disabling compactions & flushes 2024-11-24T02:36:15,449 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:15,449 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:15,449 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. after waiting 0 ms 2024-11-24T02:36:15,449 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:15,449 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:15,449 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7253aee85dc96326b8dc14f1506394b7: Waiting for close lock at 1732415775449Disabling compacts and flushes for region at 1732415775449Disabling writes for close at 1732415775449Writing region close event to WAL at 1732415775449Closed at 1732415775449 2024-11-24T02:36:15,450 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:36:15,450 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1732415775450"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415775450"}]},"ts":"1732415775450"} 2024-11-24T02:36:15,452 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:36:15,453 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:36:15,454 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415775453"}]},"ts":"1732415775453"} 2024-11-24T02:36:15,456 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-11-24T02:36:15,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, ASSIGN}] 2024-11-24T02:36:15,457 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, ASSIGN 2024-11-24T02:36:15,458 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:36:15,537 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T02:36:15,609 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7253aee85dc96326b8dc14f1506394b7, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:15,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, ASSIGN because future has completed 2024-11-24T02:36:15,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7253aee85dc96326b8dc14f1506394b7, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:15,768 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:15,769 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 7253aee85dc96326b8dc14f1506394b7, NAME => 'ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:36:15,769 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:36:15,770 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,770 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:15,770 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,770 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T02:36:15,773 INFO [StoreOpener-7253aee85dc96326b8dc14f1506394b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,775 INFO [StoreOpener-7253aee85dc96326b8dc14f1506394b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7253aee85dc96326b8dc14f1506394b7 columnFamilyName f 2024-11-24T02:36:15,775 DEBUG [StoreOpener-7253aee85dc96326b8dc14f1506394b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:15,776 INFO [StoreOpener-7253aee85dc96326b8dc14f1506394b7-1 {}] regionserver.HStore(327): Store=7253aee85dc96326b8dc14f1506394b7/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:15,776 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,778 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,779 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,779 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,780 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,782 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,785 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:36:15,785 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 7253aee85dc96326b8dc14f1506394b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10831601440, jitterRate=0.008771494030952454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T02:36:15,785 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:15,786 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 7253aee85dc96326b8dc14f1506394b7: Running coprocessor pre-open hook at 1732415775770Writing region info on filesystem at 1732415775770Initializing all the Stores at 1732415775772 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415775772Cleaning up temporary data from old regions at 1732415775780 (+8 ms)Running coprocessor post-open hooks at 1732415775785 (+5 ms)Region opened successfully at 1732415775786 (+1 ms) 2024-11-24T02:36:15,787 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., pid=69, masterSystemTime=1732415775763 2024-11-24T02:36:15,788 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:15,788 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:15,789 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7253aee85dc96326b8dc14f1506394b7, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:15,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7253aee85dc96326b8dc14f1506394b7, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:15,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-24T02:36:15,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7253aee85dc96326b8dc14f1506394b7, server=a67e7d17e593,38807,1732415667927 in 179 msec 2024-11-24T02:36:15,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-24T02:36:15,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, ASSIGN in 337 msec 2024-11-24T02:36:15,795 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:36:15,796 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415775795"}]},"ts":"1732415775795"} 2024-11-24T02:36:15,797 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-11-24T02:36:15,798 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:36:15,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:table2 in 1.1650 sec 2024-11-24T02:36:16,600 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-11-24T02:36:16,600 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2024-11-24T02:36:16,603 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-11-24T02:36:16,603 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2024-11-24T02:36:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T02:36:16,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:table2 completed 2024-11-24T02:36:16,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T02:36:16,787 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037 from hbase tables ns2:test-17324156774071 to tables ns2:table2 2024-11-24T02:36:16,787 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17324156774071 into ns2:table2 2024-11-24T02:36:16,790 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415776788 from hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:36:16,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87f9779, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:16,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:16,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:16,792 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:16,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:16,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:16,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22508570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:16,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:16,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:16,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:16,793 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:16,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d58daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:16,794 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:16,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:16,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8d9ee02 2024-11-24T02:36:16,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:16,796 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:16,796 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T02:36:16,796 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-11-24T02:36:16,797 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T02:36:16,797 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:16,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:16,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:16,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:16,800 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-11-24T02:36:16,800 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-11-24T02:36:16,800 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/partitions_eb03d854-bc37-4ce2-807b-20e12ad996f1 2024-11-24T02:36:16,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741980_1156 (size=153) 2024-11-24T02:36:17,207 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:17,208 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:17,208 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,066 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-3468876524603793799.jar 2024-11-24T02:36:18,066 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:18,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:36:18,069 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:36:18,069 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:36:18,069 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:36:18,069 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:36:18,069 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:36:18,070 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:36:18,070 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:36:18,070 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:36:18,070 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:36:18,070 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,071 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,071 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,071 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,071 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,071 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:18,071 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:36:18,071 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-11-24T02:36:18,072 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:18,072 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:18,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:18,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:18,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:18,072 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T02:36:18,072 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:18,083 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:36:18,093 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T02:36:18,176 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local718574961_0013_2ab117ee-4086-463e-9fde-a55cc19eb1ce/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:36:18,176 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local718574961_0013_2ab117ee-4086-463e-9fde-a55cc19eb1ce/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:36:18,229 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 2024-11-24T02:36:18,233 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:18,262 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f615afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:18,262 DEBUG [pool-272-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:18,262 DEBUG [pool-272-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:18,263 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:18,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:18,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:18,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@611f3c84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:18,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:18,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:18,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:18,265 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:18,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667a78e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:18,266 DEBUG [pool-272-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:18,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:18,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:18,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:18,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row0', locateType=CURRENT is [region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:18,270 INFO [pool-272-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-272-thread-1. 2024-11-24T02:36:18,271 DEBUG [pool-272-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:18,271 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:18,271 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:18,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:18,271 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-24T02:36:18,271 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:18,278 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415776788/_temporary/0/_temporary/attempt_local718574961_0013_r_000000_0/f/d352707164fc456c998e05270eb7b7c8 is 37, key is row10/f:q1/1732415681430/Put/seqid=0 2024-11-24T02:36:18,279 WARN [Thread-1720 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:18,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741981_1157 (size=8741) 2024-11-24T02:36:18,434 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6df51c7a7a1591e81897afbbac0da0ea, had cached 0 bytes from a total of 41499 2024-11-24T02:36:19,213 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415776788 2024-11-24T02:36:19,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54247727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:19,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:19,214 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:19,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:19,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:19,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b208f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:19,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:19,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,216 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:19,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@506a616a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,217 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:19,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:19,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:19,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:19,222 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415776788/_SUCCESS 2024-11-24T02:36:19,225 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:19,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:19,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16778882 2024-11-24T02:36:19,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:19,228 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:19,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:19,233 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T02:36:19,239 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415776788/f/d352707164fc456c998e05270eb7b7c8 first=Optional[row0] last=Optional[row98] 2024-11-24T02:36:19,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415776788/f/d352707164fc456c998e05270eb7b7c8 for inclusion in 7253aee85dc96326b8dc14f1506394b7/f 2024-11-24T02:36:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-24T02:36:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T02:36:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(2603): Flush status journal for 7253aee85dc96326b8dc14f1506394b7: 2024-11-24T02:36:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415776788/f/d352707164fc456c998e05270eb7b7c8 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__n83aipnhpnf2i9do8pl3i0t7pu4hlcmuuhkujsfehqbuh5oitdl8ipuvb0mqom3o/f/d352707164fc456c998e05270eb7b7c8 2024-11-24T02:36:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__n83aipnhpnf2i9do8pl3i0t7pu4hlcmuuhkujsfehqbuh5oitdl8ipuvb0mqom3o/f/d352707164fc456c998e05270eb7b7c8 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/f/03b72e0253874be294dd94c699c76d7a_SeqId_4_ 2024-11-24T02:36:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@881a558, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:19,253 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:19,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:19,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:19,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a96d30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:19,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:19,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,254 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:19,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c5c0f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:19,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:19,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f70f7aa 2024-11-24T02:36:19,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:19,257 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:19,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:19,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:19,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:19,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:19,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:19,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:19,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:19,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__n83aipnhpnf2i9do8pl3i0t7pu4hlcmuuhkujsfehqbuh5oitdl8ipuvb0mqom3o/f/d352707164fc456c998e05270eb7b7c8 into 7253aee85dc96326b8dc14f1506394b7/f as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/f/03b72e0253874be294dd94c699c76d7a_SeqId_4_ - updating store file list. 2024-11-24T02:36:19,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/f/03b72e0253874be294dd94c699c76d7a_SeqId_4_ into 7253aee85dc96326b8dc14f1506394b7/f 2024-11-24T02:36:19,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__n83aipnhpnf2i9do8pl3i0t7pu4hlcmuuhkujsfehqbuh5oitdl8ipuvb0mqom3o/f/d352707164fc456c998e05270eb7b7c8 into 7253aee85dc96326b8dc14f1506394b7/f (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/f/03b72e0253874be294dd94c699c76d7a_SeqId_4_) 2024-11-24T02:36:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__n83aipnhpnf2i9do8pl3i0t7pu4hlcmuuhkujsfehqbuh5oitdl8ipuvb0mqom3o/f/d352707164fc456c998e05270eb7b7c8 2024-11-24T02:36:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4ccb6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:19,283 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:19,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:19,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:19,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa9d749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:19,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:19,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:19,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f493b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:19,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:19,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5030a513 2024-11-24T02:36:19,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:19,287 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T02:36:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:19,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:19,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:19,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T02:36:19,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:19,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:19,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:19,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:19,307 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:19,308 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:192) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:19,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:19,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:19,308 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T02:36:19,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 , the old value is region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:19,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:19,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 from cache 2024-11-24T02:36:19,308 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-24T02:36:19,308 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1732415687521 hdfs://localhost:40215/backupUT/backup_1732415687521/ns2/test-17324156774071/ 2024-11-24T02:36:19,308 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-11-24T02:36:19,309 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T02:36:19,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:19,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:19,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:19,335 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:40215/backupUT/backup_1732415755593/.backup.manifest 2024-11-24T02:36:19,337 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732415755593/.backup.manifest 2024-11-24T02:36:19,338 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:40215/backupUT/backup_1732415722346/.backup.manifest 2024-11-24T02:36:19,340 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732415722346/.backup.manifest 2024-11-24T02:36:19,340 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'test-1732415677407' to 'table1' from full backup image hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407 2024-11-24T02:36:19,342 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'test-1732415677407', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1732415755593 2024-11-24T02:36:19,346 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'table1', preserving region splits 2024-11-24T02:36:19,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable table1 2024-11-24T02:36:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=table1 2024-11-24T02:36:19,358 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415779358"}]},"ts":"1732415779358"} 2024-11-24T02:36:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:36:19,360 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLING in hbase:meta 2024-11-24T02:36:19,360 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set table1 to state=DISABLING 2024-11-24T02:36:19,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=table1}] 2024-11-24T02:36:19,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, UNASSIGN}, {pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, UNASSIGN}] 2024-11-24T02:36:19,368 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, UNASSIGN 2024-11-24T02:36:19,368 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, UNASSIGN 2024-11-24T02:36:19,368 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=1523f0656aca8ceaf15f6bcfbf77d271, regionState=CLOSING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:19,368 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=c8065a920e0f1af7868a104494e4732d, regionState=CLOSING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:19,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, UNASSIGN because future has completed 2024-11-24T02:36:19,370 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:36:19,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1523f0656aca8ceaf15f6bcfbf77d271, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:19,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, UNASSIGN because future has completed 2024-11-24T02:36:19,371 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:36:19,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure c8065a920e0f1af7868a104494e4732d, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:19,411 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 34 more 2024-11-24T02:36:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:36:19,524 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(122): Close 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:19,524 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:36:19,525 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1722): Closing 1523f0656aca8ceaf15f6bcfbf77d271, disabling compactions & flushes 2024-11-24T02:36:19,525 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1755): Closing region table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:19,525 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:19,525 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. after waiting 0 ms 2024-11-24T02:36:19,525 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:19,536 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T02:36:19,537 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:19,538 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1973): Closed table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. 2024-11-24T02:36:19,538 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] regionserver.HRegion(1676): Region close journal for 1523f0656aca8ceaf15f6bcfbf77d271: Waiting for close lock at 1732415779524Running coprocessor pre-close hooks at 1732415779524Disabling compacts and flushes for region at 1732415779524Disabling writes for close at 1732415779525 (+1 ms)Writing region close event to WAL at 1732415779527 (+2 ms)Running coprocessor post-close hooks at 1732415779537 (+10 ms)Closed at 1732415779538 (+1 ms) 2024-11-24T02:36:19,542 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=74}] handler.UnassignRegionHandler(157): Closed 1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:19,542 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:19,542 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:36:19,542 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing c8065a920e0f1af7868a104494e4732d, disabling compactions & flushes 2024-11-24T02:36:19,542 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:19,543 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:19,543 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. after waiting 0 ms 2024-11-24T02:36:19,543 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=1523f0656aca8ceaf15f6bcfbf77d271, regionState=CLOSED 2024-11-24T02:36:19,543 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:19,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1523f0656aca8ceaf15f6bcfbf77d271, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:19,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-24T02:36:19,548 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T02:36:19,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; CloseRegionProcedure 1523f0656aca8ceaf15f6bcfbf77d271, server=a67e7d17e593,38807,1732415667927 in 177 msec 2024-11-24T02:36:19,549 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:19,549 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. 2024-11-24T02:36:19,549 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for c8065a920e0f1af7868a104494e4732d: Waiting for close lock at 1732415779542Running coprocessor pre-close hooks at 1732415779542Disabling compacts and flushes for region at 1732415779542Disabling writes for close at 1732415779543 (+1 ms)Writing region close event to WAL at 1732415779545 (+2 ms)Running coprocessor post-close hooks at 1732415779549 (+4 ms)Closed at 1732415779549 2024-11-24T02:36:19,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=1523f0656aca8ceaf15f6bcfbf77d271, UNASSIGN in 182 msec 2024-11-24T02:36:19,551 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:19,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=c8065a920e0f1af7868a104494e4732d, regionState=CLOSED 2024-11-24T02:36:19,552 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2024-11-24T02:36:19,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure c8065a920e0f1af7868a104494e4732d, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:19,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=72 2024-11-24T02:36:19,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=72, state=SUCCESS, hasLock=false; CloseRegionProcedure c8065a920e0f1af7868a104494e4732d, server=a67e7d17e593,38807,1732415667927 in 182 msec 2024-11-24T02:36:19,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-11-24T02:36:19,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=c8065a920e0f1af7868a104494e4732d, UNASSIGN in 188 msec 2024-11-24T02:36:19,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T02:36:19,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=table1 in 196 msec 2024-11-24T02:36:19,559 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415779559"}]},"ts":"1732415779559"} 2024-11-24T02:36:19,560 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=DISABLED in hbase:meta 2024-11-24T02:36:19,560 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set table1 to state=DISABLED 2024-11-24T02:36:19,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DisableTableProcedure table=table1 in 210 msec 2024-11-24T02:36:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:36:19,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:table1 completed 2024-11-24T02:36:19,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.2 truncate table1 2024-11-24T02:36:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) 2024-11-24T02:36:19,688 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'table1' regions in transition 2024-11-24T02:36:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:36:19,694 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:19,694 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:19,697 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/f, FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/f3, FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/mob, FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/recovered.edits] 2024-11-24T02:36:19,697 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/f, FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/f3, FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/mob, FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/recovered.edits] 2024-11-24T02:36:19,700 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/f/295bc44d42b74e30a82fec1b9e01d521_SeqId_4_ to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/archive/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/f/295bc44d42b74e30a82fec1b9e01d521_SeqId_4_ 2024-11-24T02:36:19,701 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/f3/57f7d99b580748429edb5e89094f9f1c_SeqId_4_ to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/archive/data/default/table1/c8065a920e0f1af7868a104494e4732d/f3/57f7d99b580748429edb5e89094f9f1c_SeqId_4_ 2024-11-24T02:36:19,703 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/mob/39cd543bd788428ab64249d09a124c2c_SeqId_4_ to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/archive/data/default/table1/c8065a920e0f1af7868a104494e4732d/mob/39cd543bd788428ab64249d09a124c2c_SeqId_4_ 2024-11-24T02:36:19,704 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/recovered.edits/6.seqid to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/archive/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271/recovered.edits/6.seqid 2024-11-24T02:36:19,705 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/1523f0656aca8ceaf15f6bcfbf77d271 2024-11-24T02:36:19,706 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d/recovered.edits/6.seqid to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/archive/data/default/table1/c8065a920e0f1af7868a104494e4732d/recovered.edits/6.seqid 2024-11-24T02:36:19,706 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/c8065a920e0f1af7868a104494e4732d 2024-11-24T02:36:19,706 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived table1 regions 2024-11-24T02:36:19,707 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-11-24T02:36:19,708 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f/mob] 2024-11-24T02:36:19,709 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/mobdir/data/default/table1/b4aaa492ee713125ae5609f2682a2d2f 2024-11-24T02:36:19,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38807 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T02:36:19,719 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 2 rows of table1 from hbase:meta 2024-11-24T02:36:19,722 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'table1' descriptor. 2024-11-24T02:36:19,723 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'table1' from region states. 2024-11-24T02:36:19,723 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,,1732415768837.c8065a920e0f1af7868a104494e4732d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732415779723"}]},"ts":"9223372036854775807"} 2024-11-24T02:36:19,723 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732415779723"}]},"ts":"9223372036854775807"} 2024-11-24T02:36:19,725 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 2 regions from META 2024-11-24T02:36:19,725 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c8065a920e0f1af7868a104494e4732d, NAME => 'table1,,1732415768837.c8065a920e0f1af7868a104494e4732d.', STARTKEY => '', ENDKEY => 'row0'}, {ENCODED => 1523f0656aca8ceaf15f6bcfbf77d271, NAME => 'table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271.', STARTKEY => 'row0', ENDKEY => ''}] 2024-11-24T02:36:19,725 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'table1' as deleted. 2024-11-24T02:36:19,726 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732415779725"}]},"ts":"9223372036854775807"} 2024-11-24T02:36:19,727 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table table1 state from META 2024-11-24T02:36:19,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741982_1158 (size=938) 2024-11-24T02:36:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:36:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:36:20,140 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3c2696984d354e614e3baf276bf25d8, NAME => 'table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8.', STARTKEY => '', ENDKEY => 'row0'}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:36:20,142 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6b1451007b9af7358fedf38311721e6c, NAME => 'table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c.', STARTKEY => 'row0', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:36:20,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741983_1159 (size=45) 2024-11-24T02:36:20,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741984_1160 (size=45) 2024-11-24T02:36:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:36:20,554 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:20,554 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(898): Instantiated table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:20,555 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1722): Closing 6b1451007b9af7358fedf38311721e6c, disabling compactions & flushes 2024-11-24T02:36:20,555 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing d3c2696984d354e614e3baf276bf25d8, disabling compactions & flushes 2024-11-24T02:36:20,555 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:20,555 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1755): Closing region table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:20,555 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:20,555 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:20,555 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. after waiting 0 ms 2024-11-24T02:36:20,555 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. after waiting 0 ms 2024-11-24T02:36:20,555 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:20,555 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:20,556 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:20,556 INFO [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1973): Closed table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:20,556 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3c2696984d354e614e3baf276bf25d8: Waiting for close lock at 1732415780555Disabling compacts and flushes for region at 1732415780555Disabling writes for close at 1732415780555Writing region close event to WAL at 1732415780556 (+1 ms)Closed at 1732415780556 2024-11-24T02:36:20,556 DEBUG [RegionOpenAndInit-table1-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6b1451007b9af7358fedf38311721e6c: Waiting for close lock at 1732415780555Disabling compacts and flushes for region at 1732415780555Disabling writes for close at 1732415780555Writing region close event to WAL at 1732415780556 (+1 ms)Closed at 1732415780556 2024-11-24T02:36:20,560 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732415780559"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415780559"}]},"ts":"1732415780559"} 2024-11-24T02:36:20,561 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732415780559"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415780559"}]},"ts":"1732415780559"} 2024-11-24T02:36:20,564 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 2 regions to meta. 2024-11-24T02:36:20,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415780565"}]},"ts":"1732415780565"} 2024-11-24T02:36:20,566 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-24T02:36:20,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, ASSIGN}, {pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, ASSIGN}] 2024-11-24T02:36:20,568 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, ASSIGN 2024-11-24T02:36:20,568 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, ASSIGN 2024-11-24T02:36:20,568 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:36:20,568 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:36:20,720 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=6b1451007b9af7358fedf38311721e6c, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:20,720 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=d3c2696984d354e614e3baf276bf25d8, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:20,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, ASSIGN because future has completed 2024-11-24T02:36:20,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:20,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, ASSIGN because future has completed 2024-11-24T02:36:20,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:36:20,887 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:20,887 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b1451007b9af7358fedf38311721e6c, NAME => 'table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-24T02:36:20,888 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:36:20,888 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,889 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:20,889 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,889 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,892 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,895 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b1451007b9af7358fedf38311721e6c columnFamilyName f 2024-11-24T02:36:20,895 DEBUG [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:20,896 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(327): Store=6b1451007b9af7358fedf38311721e6c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:20,896 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,897 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b1451007b9af7358fedf38311721e6c columnFamilyName f3 2024-11-24T02:36:20,897 DEBUG [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:20,898 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(327): Store=6b1451007b9af7358fedf38311721e6c/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:20,898 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,900 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b1451007b9af7358fedf38311721e6c columnFamilyName mob 2024-11-24T02:36:20,901 DEBUG [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:20,902 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(327): Store=6b1451007b9af7358fedf38311721e6c/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:20,903 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,903 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,904 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,905 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,905 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,905 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T02:36:20,907 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,909 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:36:20,909 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 6b1451007b9af7358fedf38311721e6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9457109440, jitterRate=-0.11923804879188538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T02:36:20,909 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:20,910 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 6b1451007b9af7358fedf38311721e6c: Running coprocessor pre-open hook at 1732415780889Writing region info on filesystem at 1732415780889Initializing all the Stores at 1732415780891 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415780891Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415780892 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415780892Cleaning up temporary data from old regions at 1732415780905 (+13 ms)Running coprocessor post-open hooks at 1732415780909 (+4 ms)Region opened successfully at 1732415780910 (+1 ms) 2024-11-24T02:36:20,911 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c., pid=79, masterSystemTime=1732415780879 2024-11-24T02:36:20,913 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:20,913 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:20,913 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:20,913 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => d3c2696984d354e614e3baf276bf25d8, NAME => 'table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-24T02:36:20,914 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=6b1451007b9af7358fedf38311721e6c, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:20,914 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:36:20,914 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,914 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:20,914 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,914 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:20,916 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,918 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c2696984d354e614e3baf276bf25d8 columnFamilyName f 2024-11-24T02:36:20,918 DEBUG [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:20,919 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(327): Store=d3c2696984d354e614e3baf276bf25d8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:20,919 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-24T02:36:20,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927 in 192 msec 2024-11-24T02:36:20,920 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c2696984d354e614e3baf276bf25d8 columnFamilyName f3 2024-11-24T02:36:20,920 DEBUG [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:20,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=76, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, ASSIGN in 352 msec 2024-11-24T02:36:20,921 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(327): Store=d3c2696984d354e614e3baf276bf25d8/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:20,921 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,922 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c2696984d354e614e3baf276bf25d8 columnFamilyName mob 2024-11-24T02:36:20,922 DEBUG [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:20,923 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(327): Store=d3c2696984d354e614e3baf276bf25d8/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:20,923 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,924 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,925 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,926 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,926 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,927 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T02:36:20,928 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,930 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:36:20,931 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened d3c2696984d354e614e3baf276bf25d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10204214720, jitterRate=-0.04965844750404358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T02:36:20,931 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:20,931 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for d3c2696984d354e614e3baf276bf25d8: Running coprocessor pre-open hook at 1732415780914Writing region info on filesystem at 1732415780914Initializing all the Stores at 1732415780915 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415780915Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415780916 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415780916Cleaning up temporary data from old regions at 1732415780926 (+10 ms)Running coprocessor post-open hooks at 1732415780931 (+5 ms)Region opened successfully at 1732415780931 2024-11-24T02:36:20,932 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., pid=80, masterSystemTime=1732415780879 2024-11-24T02:36:20,933 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:20,933 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:20,934 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=77 updating hbase:meta row=d3c2696984d354e614e3baf276bf25d8, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:20,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=77, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:20,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=77 2024-11-24T02:36:20,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=77, state=SUCCESS, hasLock=false; OpenRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927 in 209 msec 2024-11-24T02:36:20,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-24T02:36:20,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, ASSIGN in 371 msec 2024-11-24T02:36:20,940 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415780940"}]},"ts":"1732415780940"} 2024-11-24T02:36:20,942 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-24T02:36:20,942 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'table1' completed 2024-11-24T02:36:20,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=table1 preserveSplits=true) in 1.2620 sec 2024-11-24T02:36:21,273 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T02:36:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:36:21,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: default:table1 completed 2024-11-24T02:36:21,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:21,851 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315,hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79 from hbase tables test-1732415677407 to tables table1 2024-11-24T02:36:21,851 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1732415677407 into table1 2024-11-24T02:36:21,853 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851 from hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315,hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79 2024-11-24T02:36:21,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a4f4fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:21,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:21,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:21,855 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:21,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:21,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:21,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@125d7c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:21,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:21,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:21,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:21,857 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:21,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5a5cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:21,858 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:21,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:21,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3592803d 2024-11-24T02:36:21,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:21,859 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:21,860 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T02:36:21,860 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-24T02:36:21,861 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:21,861 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:21,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:21,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:21,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:21,865 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-24T02:36:21,865 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-24T02:36:21,865 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-24T02:36:21,865 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/partitions_ce6e2876-7609-4522-900c-3cd3f2e906ff 2024-11-24T02:36:21,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741985_1161 (size=177) 2024-11-24T02:36:22,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:22,277 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:22,277 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-17157586836796851771.jar 2024-11-24T02:36:23,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:23,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:36:23,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:36:23,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:36:23,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:36:23,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:36:23,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:36:23,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:36:23,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:36:23,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:36:23,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:36:23,138 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,138 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,138 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,138 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,138 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,139 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:23,139 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:36:23,139 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-24T02:36:23,139 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:23,139 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:23,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:23,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:23,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:23,139 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T02:36:23,140 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:23,149 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:36:23,158 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T02:36:23,243 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1537220336_0014_68549f2c-a3f3-42eb-ad58-c370fa216bd8/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:36:23,243 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1537220336_0014_68549f2c-a3f3-42eb-ad58-c370fa216bd8/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:36:23,292 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/5d3f9bfaddac4bf4a79e8d16491044f6 2024-11-24T02:36:23,294 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:23,336 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/30c452fa45f2462f8bdd8c1b3f2c0f4e 2024-11-24T02:36:23,339 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:23,362 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f/53a13e59c6ae40e384cc3ce799783392 2024-11-24T02:36:23,365 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:23,394 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/b292982b47f4868077ccf0678ba72a79/mob/d41d8cd98f00b204e9800998ecf8427e202411245e8ddfb02180478584792eab2f3b28f1_a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:36:23,397 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:23,418 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415722346/default/test-1732415677407/archive/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/mob/d188d82f983547d6962f28b22a0c0e2c 2024-11-24T02:36:23,420 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:23,452 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458f2ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:23,452 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:23,452 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:23,453 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:23,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:23,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:23,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7b3907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:23,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:23,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:23,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:23,455 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:23,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a41056a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:23,456 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:23,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:23,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:23,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:23,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732415677407-1-0', locateType=CURRENT is [region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:23,461 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-24T02:36:23,461 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:23,461 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:23,461 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:23,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:23,461 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732415677407-1-0] 2024-11-24T02:36:23,461 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:23,472 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6741bddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:23,472 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:23,472 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:23,473 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:23,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:23,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:23,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13eba4d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:23,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:23,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:23,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:23,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:23,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32166d20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:23,475 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:23,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:23,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:23,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:23,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732415677407-3-0', locateType=CURRENT is [region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:23,480 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-24T02:36:23,480 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:23,480 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:23,480 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:23,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:23,480 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732415677407-3-0] 2024-11-24T02:36:23,480 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:23,485 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/_temporary/0/_temporary/attempt_local1537220336_0014_r_000000_0/f/9bafd1126b2046c583d824fee30eeb90 is 59, key is row-test-1732415677407-1-10/f:q1/1732415706753/Put/seqid=0 2024-11-24T02:36:23,487 WARN [Thread-1807 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:23,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741986_1162 (size=11125) 2024-11-24T02:36:23,891 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/_temporary/0/_temporary/attempt_local1537220336_0014_r_000000_0/mob/e57017889964496682b754ce06622a8d is 192, key is row-test-1732415677407-3-100/mob:q1/1732415707081/Put/seqid=0 2024-11-24T02:36:23,894 WARN [Thread-1809 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:23,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741987_1163 (size=13404) 2024-11-24T02:36:24,327 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9d83f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:24,327 DEBUG [pool-277-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:24,327 DEBUG [pool-277-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:24,328 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:24,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:24,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:24,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d42f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:24,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:24,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:24,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:24,329 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:24,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d605190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:24,330 DEBUG [pool-277-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:24,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:24,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:24,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:24,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:24,335 INFO [pool-277-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-277-thread-1. 2024-11-24T02:36:24,335 DEBUG [pool-277-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:24,335 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:24,336 DEBUG [pool-277-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:24,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:24,336 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row0] 2024-11-24T02:36:24,336 DEBUG [pool-277-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:24,340 DEBUG [pool-277-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/_temporary/0/_temporary/attempt_local1537220336_0014_r_000001_0/f/b63efcdcf4094d5084acd593193a969a is 37, key is row10/f:q1/1732415680115/Put/seqid=0 2024-11-24T02:36:24,342 WARN [Thread-1824 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:24,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741988_1164 (size=8742) 2024-11-24T02:36:25,161 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:36:25,291 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851 2024-11-24T02:36:25,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42974c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:25,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:25,292 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:25,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:25,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:25,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c20e5bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:25,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:25,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:25,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6bbb0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:25,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:25,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:25,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:25,299 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/_SUCCESS 2024-11-24T02:36:25,305 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:25,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:25,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c9b1a12 2024-11-24T02:36:25,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:25,308 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:25,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:25,313 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,318 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/f/9bafd1126b2046c583d824fee30eeb90 first=Optional[row-test-1732415677407-1-0] last=Optional[row-test-1732415677407-1-98] 2024-11-24T02:36:25,321 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/f/b63efcdcf4094d5084acd593193a969a first=Optional[row0] last=Optional[row98] 2024-11-24T02:36:25,323 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/mob/e57017889964496682b754ce06622a8d first=Optional[row-test-1732415677407-3-0] last=Optional[row-test-1732415677407-3-99] 2024-11-24T02:36:25,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/f/b63efcdcf4094d5084acd593193a969a for inclusion in 6b1451007b9af7358fedf38311721e6c/f 2024-11-24T02:36:25,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/f/9bafd1126b2046c583d824fee30eeb90 for inclusion in d3c2696984d354e614e3baf276bf25d8/f 2024-11-24T02:36:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row0 last=row98 2024-11-24T02:36:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first=row0 last= 2024-11-24T02:36:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HRegion(2603): Flush status journal for 6b1451007b9af7358fedf38311721e6c: 2024-11-24T02:36:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/f/b63efcdcf4094d5084acd593193a969a to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/b63efcdcf4094d5084acd593193a969a 2024-11-24T02:36:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732415677407-1-0 last=row-test-1732415677407-1-98 2024-11-24T02:36:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T02:36:25,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/mob/e57017889964496682b754ce06622a8d for inclusion in d3c2696984d354e614e3baf276bf25d8/mob 2024-11-24T02:36:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/b63efcdcf4094d5084acd593193a969a as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c/f/96256f18c776400da667cf7975d1be51_SeqId_4_ 2024-11-24T02:36:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9fed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:25,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:25,336 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:25,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:25,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732415677407-3-0 last=row-test-1732415677407-3-99 2024-11-24T02:36:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T02:36:25,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1174090c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(2603): Flush status journal for d3c2696984d354e614e3baf276bf25d8: 2024-11-24T02:36:25,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:25,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,337 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/f/9bafd1126b2046c583d824fee30eeb90 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/9bafd1126b2046c583d824fee30eeb90 2024-11-24T02:36:25,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aac8f38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:25,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:25,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@158b9979 2024-11-24T02:36:25,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/9bafd1126b2046c583d824fee30eeb90 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f/49df2a0e3beb42a1b802bb448b69644b_SeqId_4_ 2024-11-24T02:36:25,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415781851/mob/e57017889964496682b754ce06622a8d to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/mob/e57017889964496682b754ce06622a8d 2024-11-24T02:36:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:25,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:25,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:25,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/mob/e57017889964496682b754ce06622a8d as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/mob/f0ba2b613357429fb9abecd3c22897c0_SeqId_4_ 2024-11-24T02:36:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d8d18ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:25,345 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:25,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:25,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:25,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60de2fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:25,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:25,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,347 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:25,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba63e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:25,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:25,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a8ee8e8 2024-11-24T02:36:25,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:25,350 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:25,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:25,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:25,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:25,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:25,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807. 2024-11-24T02:36:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:25,364 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/b63efcdcf4094d5084acd593193a969a into 6b1451007b9af7358fedf38311721e6c/f as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c/f/96256f18c776400da667cf7975d1be51_SeqId_4_ - updating store file list. 2024-11-24T02:36:25,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c/f/96256f18c776400da667cf7975d1be51_SeqId_4_ into 6b1451007b9af7358fedf38311721e6c/f 2024-11-24T02:36:25,370 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/b63efcdcf4094d5084acd593193a969a into 6b1451007b9af7358fedf38311721e6c/f (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c/f/96256f18c776400da667cf7975d1be51_SeqId_4_) 2024-11-24T02:36:25,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/b63efcdcf4094d5084acd593193a969a 2024-11-24T02:36:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ff8760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:25,372 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:25,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:25,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:25,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3967d208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:25,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:25,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:25,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,374 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:25,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bd4eaf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aed66b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:25,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:25,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5804bd14 2024-11-24T02:36:25,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:25,377 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:25,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:25,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:25,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78efd06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:25,377 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T02:36:25,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:25,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:25,378 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:25,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79530837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:25,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:25,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:25,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:25,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3dd00ec3 2024-11-24T02:36:25,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:25,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T02:36:25,380 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:25,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:25,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:25,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:25,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:25,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,394 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807. 2024-11-24T02:36:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:25,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:25,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:25,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/9bafd1126b2046c583d824fee30eeb90 into d3c2696984d354e614e3baf276bf25d8/f as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f/49df2a0e3beb42a1b802bb448b69644b_SeqId_4_ - updating store file list. 2024-11-24T02:36:25,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f/49df2a0e3beb42a1b802bb448b69644b_SeqId_4_ into d3c2696984d354e614e3baf276bf25d8/f 2024-11-24T02:36:25,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/9bafd1126b2046c583d824fee30eeb90 into d3c2696984d354e614e3baf276bf25d8/f (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f/49df2a0e3beb42a1b802bb448b69644b_SeqId_4_) 2024-11-24T02:36:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/f/9bafd1126b2046c583d824fee30eeb90 2024-11-24T02:36:25,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/mob/e57017889964496682b754ce06622a8d into d3c2696984d354e614e3baf276bf25d8/mob as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/mob/f0ba2b613357429fb9abecd3c22897c0_SeqId_4_ - updating store file list. 2024-11-24T02:36:25,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/mob/f0ba2b613357429fb9abecd3c22897c0_SeqId_4_ into d3c2696984d354e614e3baf276bf25d8/mob 2024-11-24T02:36:25,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/mob/e57017889964496682b754ce06622a8d into d3c2696984d354e614e3baf276bf25d8/mob (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/mob/f0ba2b613357429fb9abecd3c22897c0_SeqId_4_) 2024-11-24T02:36:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__k10t30nl8qcpdl8f9gdvcrr948mdjo4ku8h80bdjfd5osg8ipgtbum6i8rlb02im/mob/e57017889964496682b754ce06622a8d 2024-11-24T02:36:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c5d2f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:25,416 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:25,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:25,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:25,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5918d7a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:25,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:25,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,417 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:25,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d33d33f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:25,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:25,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f88bb48 2024-11-24T02:36:25,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:25,420 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T02:36:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:25,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:25,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:25,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T02:36:25,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:25,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:25,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:25,436 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:25,436 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:25,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:25,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:25,436 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T02:36:25,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 , the old value is region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:25,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:25,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 from cache 2024-11-24T02:36:25,437 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T02:36:25,456 INFO [Time-limited test {}] impl.RestoreTablesClient(181): Restoring 'test-1732415677407' to 'table1' from log dirs: hdfs://localhost:40215/backupUT/backup_1732415755593/default/test-1732415677407/f2/a39a2599bbd44757a26c9170f1156673 2024-11-24T02:36:25,459 DEBUG [Time-limited test {}] util.RestoreTool(177): Found descriptor 'test-1732415677407', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} through backup_1732415755593 2024-11-24T02:36:25,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table table1 from 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=table1 2024-11-24T02:36:25,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741989_1165 (size=938) 2024-11-24T02:36:25,542 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 34 more 2024-11-24T02:36:25,554 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-24T02:36:25,877 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/.tabledesc/.tableinfo.0000000001.938 2024-11-24T02:36:25,877 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/.tabledesc/.tableinfo.0000000002.938 2024-11-24T02:36:25,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=table1}] 2024-11-24T02:36:25,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, REOPEN/MOVE}, {pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, REOPEN/MOVE}] 2024-11-24T02:36:25,886 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, REOPEN/MOVE 2024-11-24T02:36:25,886 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, REOPEN/MOVE 2024-11-24T02:36:25,887 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=6b1451007b9af7358fedf38311721e6c, regionState=CLOSING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:25,887 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=d3c2696984d354e614e3baf276bf25d8, regionState=CLOSING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:25,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, REOPEN/MOVE because future has completed 2024-11-24T02:36:25,890 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:36:25,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:25,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, REOPEN/MOVE because future has completed 2024-11-24T02:36:25,890 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:36:25,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:26,042 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,042 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:36:26,043 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 6b1451007b9af7358fedf38311721e6c, disabling compactions & flushes 2024-11-24T02:36:26,043 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:26,043 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:26,043 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. after waiting 0 ms 2024-11-24T02:36:26,043 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:26,049 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T02:36:26,050 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:26,050 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:26,050 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 6b1451007b9af7358fedf38311721e6c: Waiting for close lock at 1732415786042Running coprocessor pre-close hooks at 1732415786042Disabling compacts and flushes for region at 1732415786042Disabling writes for close at 1732415786043 (+1 ms)Writing region close event to WAL at 1732415786044 (+1 ms)Running coprocessor post-close hooks at 1732415786050 (+6 ms)Closed at 1732415786050 2024-11-24T02:36:26,051 WARN [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 6b1451007b9af7358fedf38311721e6c to self. 2024-11-24T02:36:26,053 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,053 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(122): Close d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,053 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:36:26,054 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1722): Closing d3c2696984d354e614e3baf276bf25d8, disabling compactions & flushes 2024-11-24T02:36:26,054 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1755): Closing region table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:26,054 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:26,054 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1843): Acquired close lock on table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. after waiting 0 ms 2024-11-24T02:36:26,054 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1853): Updates disabled for region table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:26,054 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=6b1451007b9af7358fedf38311721e6c, regionState=CLOSED 2024-11-24T02:36:26,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:26,060 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T02:36:26,061 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:26,061 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1973): Closed table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:26,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-24T02:36:26,061 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegion(1676): Region close journal for d3c2696984d354e614e3baf276bf25d8: Waiting for close lock at 1732415786053Running coprocessor pre-close hooks at 1732415786053Disabling compacts and flushes for region at 1732415786054 (+1 ms)Disabling writes for close at 1732415786054Writing region close event to WAL at 1732415786055 (+1 ms)Running coprocessor post-close hooks at 1732415786061 (+6 ms)Closed at 1732415786061 2024-11-24T02:36:26,061 WARN [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] regionserver.HRegionServer(3299): Not adding moved region record: d3c2696984d354e614e3baf276bf25d8 to self. 2024-11-24T02:36:26,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927 in 168 msec 2024-11-24T02:36:26,062 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, REOPEN/MOVE; state=CLOSED, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=true 2024-11-24T02:36:26,064 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=86}] handler.UnassignRegionHandler(157): Closed d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=d3c2696984d354e614e3baf276bf25d8, regionState=CLOSED 2024-11-24T02:36:26,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=83, state=RUNNABLE, hasLock=false; CloseRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:26,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=83 2024-11-24T02:36:26,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=83, state=SUCCESS, hasLock=false; CloseRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927 in 176 msec 2024-11-24T02:36:26,070 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, REOPEN/MOVE; state=CLOSED, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=true 2024-11-24T02:36:26,213 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=d3c2696984d354e614e3baf276bf25d8, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:26,213 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=6b1451007b9af7358fedf38311721e6c, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:26,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, REOPEN/MOVE because future has completed 2024-11-24T02:36:26,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:26,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, REOPEN/MOVE because future has completed 2024-11-24T02:36:26,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:26,377 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:26,378 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b1451007b9af7358fedf38311721e6c, NAME => 'table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c.', STARTKEY => 'row0', ENDKEY => ''} 2024-11-24T02:36:26,378 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:36:26,379 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,379 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:26,379 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,379 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,382 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,383 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b1451007b9af7358fedf38311721e6c columnFamilyName f 2024-11-24T02:36:26,384 DEBUG [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:26,390 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T02:36:26,391 DEBUG [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c/f/96256f18c776400da667cf7975d1be51_SeqId_4_ 2024-11-24T02:36:26,391 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(327): Store=6b1451007b9af7358fedf38311721e6c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:26,391 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,392 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b1451007b9af7358fedf38311721e6c columnFamilyName f2 2024-11-24T02:36:26,392 DEBUG [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:26,393 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(327): Store=6b1451007b9af7358fedf38311721e6c/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:26,393 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,394 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b1451007b9af7358fedf38311721e6c columnFamilyName mob 2024-11-24T02:36:26,395 DEBUG [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:26,395 INFO [StoreOpener-6b1451007b9af7358fedf38311721e6c-1 {}] regionserver.HStore(327): Store=6b1451007b9af7358fedf38311721e6c/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:26,395 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,396 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,397 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,398 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,398 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,398 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T02:36:26,399 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,400 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 6b1451007b9af7358fedf38311721e6c; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11392061600, jitterRate=0.060968413949012756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T02:36:26,400 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:26,401 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 6b1451007b9af7358fedf38311721e6c: Running coprocessor pre-open hook at 1732415786379Writing region info on filesystem at 1732415786379Initializing all the Stores at 1732415786381 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415786381Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415786381Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415786381Cleaning up temporary data from old regions at 1732415786398 (+17 ms)Running coprocessor post-open hooks at 1732415786400 (+2 ms)Region opened successfully at 1732415786401 (+1 ms) 2024-11-24T02:36:26,402 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c., pid=88, masterSystemTime=1732415786372 2024-11-24T02:36:26,404 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:26,404 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:26,404 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:26,404 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => d3c2696984d354e614e3baf276bf25d8, NAME => 'table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8.', STARTKEY => '', ENDKEY => 'row0'} 2024-11-24T02:36:26,404 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=6b1451007b9af7358fedf38311721e6c, regionState=OPEN, openSeqNum=7, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:26,405 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:36:26,405 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,405 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:26,405 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,405 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:26,408 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,409 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c2696984d354e614e3baf276bf25d8 columnFamilyName f 2024-11-24T02:36:26,409 DEBUG [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:26,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=84 2024-11-24T02:36:26,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b1451007b9af7358fedf38311721e6c, server=a67e7d17e593,38807,1732415667927 in 187 msec 2024-11-24T02:36:26,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=6b1451007b9af7358fedf38311721e6c, REOPEN/MOVE in 525 msec 2024-11-24T02:36:26,414 DEBUG [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f/49df2a0e3beb42a1b802bb448b69644b_SeqId_4_ 2024-11-24T02:36:26,414 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(327): Store=d3c2696984d354e614e3baf276bf25d8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:26,415 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,416 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c2696984d354e614e3baf276bf25d8 columnFamilyName f2 2024-11-24T02:36:26,416 DEBUG [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:26,416 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(327): Store=d3c2696984d354e614e3baf276bf25d8/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:26,417 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,417 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c2696984d354e614e3baf276bf25d8 columnFamilyName mob 2024-11-24T02:36:26,418 DEBUG [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:26,423 DEBUG [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/mob/f0ba2b613357429fb9abecd3c22897c0_SeqId_4_ 2024-11-24T02:36:26,423 INFO [StoreOpener-d3c2696984d354e614e3baf276bf25d8-1 {}] regionserver.HStore(327): Store=d3c2696984d354e614e3baf276bf25d8/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:26,423 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,424 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,425 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,426 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,426 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,426 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table table1 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T02:36:26,428 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,428 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened d3c2696984d354e614e3baf276bf25d8; next sequenceid=7; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11245532160, jitterRate=0.04732179641723633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T02:36:26,428 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:26,428 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for d3c2696984d354e614e3baf276bf25d8: Running coprocessor pre-open hook at 1732415786405Writing region info on filesystem at 1732415786405Initializing all the Stores at 1732415786406 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415786406Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415786407 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415786408 (+1 ms)Cleaning up temporary data from old regions at 1732415786426 (+18 ms)Running coprocessor post-open hooks at 1732415786428 (+2 ms)Region opened successfully at 1732415786428 2024-11-24T02:36:26,429 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., pid=87, masterSystemTime=1732415786372 2024-11-24T02:36:26,431 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:26,431 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:26,431 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=83 updating hbase:meta row=d3c2696984d354e614e3baf276bf25d8, regionState=OPEN, openSeqNum=7, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:26,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=83, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:26,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=83 2024-11-24T02:36:26,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=83, state=SUCCESS, hasLock=false; OpenRegionProcedure d3c2696984d354e614e3baf276bf25d8, server=a67e7d17e593,38807,1732415667927 in 215 msec 2024-11-24T02:36:26,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-24T02:36:26,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=d3c2696984d354e614e3baf276bf25d8, REOPEN/MOVE in 550 msec 2024-11-24T02:36:26,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-24T02:36:26,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=table1 in 555 msec 2024-11-24T02:36:26,439 DEBUG [PEWorker-2 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=table1 2024-11-24T02:36:26,440 DEBUG [PEWorker-2 {}] backup.HFileArchiver(265): No files to dispose of in table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., family=f3 2024-11-24T02:36:26,441 DEBUG [PEWorker-2 {}] backup.HFileArchiver(265): No files to dispose of in table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c., family=f3 2024-11-24T02:36:26,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; ModifyTableProcedure table=table1 in 979 msec 2024-11-24T02:36:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-24T02:36:26,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:table1 completed 2024-11-24T02:36:26,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:26,447 INFO [Time-limited test {}] util.RestoreTool(200): Changed table1 to: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:26,447 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore incremental backup from directory hdfs://localhost:40215/backupUT/backup_1732415755593/default/test-1732415677407/f2/a39a2599bbd44757a26c9170f1156673 from hbase tables test-1732415677407 to tables table1 2024-11-24T02:36:26,447 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore test-1732415677407 into table1 2024-11-24T02:36:26,448 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415786447 from hdfs://localhost:40215/backupUT/backup_1732415755593/default/test-1732415677407/f2/a39a2599bbd44757a26c9170f1156673 2024-11-24T02:36:26,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@507d23f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:26,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:26,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:26,449 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:26,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:26,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:26,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40a9ceb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:26,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:26,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:26,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:26,451 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:26,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7c1a06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:26,451 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:26,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:26,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@340acba1 2024-11-24T02:36:26,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:26,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:26,453 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T02:36:26,454 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table table1 2024-11-24T02:36:26,454 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:26,454 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:26,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:26,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:26,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:26,458 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: 2024-11-24T02:36:26,458 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for table1: row0 2024-11-24T02:36:26,458 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-24T02:36:26,458 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/partitions_503494cd-9bed-4257-a3db-82aec7f90e8b 2024-11-24T02:36:26,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741990_1166 (size=177) 2024-11-24T02:36:26,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:26,866 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:26,866 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-5116102588369842726.jar 2024-11-24T02:36:27,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:27,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:36:27,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:36:27,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:36:27,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:36:27,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:36:27,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:36:27,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:36:27,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:36:27,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:36:27,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:36:27,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,729 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:27,729 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:36:27,729 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: table1 2024-11-24T02:36:27,729 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:27,729 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:27,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:27,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:27,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:27,729 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T02:36:27,730 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:27,739 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:36:27,750 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T02:36:27,823 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1342616579_0015_4927dc0a-a8a2-4bcb-836b-eaa75df9927a/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:36:27,823 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local1342616579_0015_4927dc0a-a8a2-4bcb-836b-eaa75df9927a/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:36:27,878 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415755593/default/test-1732415677407/f2/a39a2599bbd44757a26c9170f1156673 2024-11-24T02:36:27,881 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:27,911 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b7d9ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:27,911 DEBUG [pool-281-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:27,911 DEBUG [pool-281-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:27,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:27,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:27,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:27,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55139f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:27,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:27,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:27,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:27,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:27,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ab8afd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:27,914 DEBUG [pool-281-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:27,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:27,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:27,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:27,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row-test-1732415677407-2-0', locateType=CURRENT is [region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=7] 2024-11-24T02:36:27,918 INFO [pool-281-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-281-thread-1. 2024-11-24T02:36:27,918 DEBUG [pool-281-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:27,918 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:27,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:27,918 DEBUG [pool-281-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:27,919 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732415677407-2-0] 2024-11-24T02:36:27,919 DEBUG [pool-281-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:27,922 DEBUG [pool-281-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415786447/_temporary/0/_temporary/attempt_local1342616579_0015_r_000000_0/f2/856bbf0af8a9445cb67ab54889e342c4 is 58, key is row-test-1732415677407-2-0/f2:q1/1732415750583/Put/seqid=0 2024-11-24T02:36:27,924 WARN [Thread-1888 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:27,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741991_1167 (size=5609) 2024-11-24T02:36:28,867 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415786447 2024-11-24T02:36:28,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68eb29fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:28,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:28,869 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:28,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:28,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:28,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a874d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:28,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:28,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,870 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:28,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cac950a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:28,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:28,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:28,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:28,876 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415786447/_SUCCESS 2024-11-24T02:36:28,879 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:28,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:28,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@74d8ad19 2024-11-24T02:36:28,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:28,881 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:28,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=7] 2024-11-24T02:36:28,886 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=table1,, stopping at row=table1 ,, for max=2147483647 with caching=100 2024-11-24T02:36:28,892 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415786447/f2/856bbf0af8a9445cb67ab54889e342c4 first=Optional[row-test-1732415677407-2-0] last=Optional[row-test-1732415677407-2-6] 2024-11-24T02:36:28,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415786447/f2/856bbf0af8a9445cb67ab54889e342c4 for inclusion in d3c2696984d354e614e3baf276bf25d8/f2 2024-11-24T02:36:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row-test-1732415677407-2-0 last=row-test-1732415677407-2-6 2024-11-24T02:36:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first= last=row0 2024-11-24T02:36:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(2603): Flush status journal for d3c2696984d354e614e3baf276bf25d8: 2024-11-24T02:36:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-default-table1-1732415786447/f2/856bbf0af8a9445cb67ab54889e342c4 to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__rbnuc46iihvhfluvnkhulrdf86kv3jj9r9n7c6onruq9oqf271s455d526g3gtev/f2/856bbf0af8a9445cb67ab54889e342c4 2024-11-24T02:36:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__rbnuc46iihvhfluvnkhulrdf86kv3jj9r9n7c6onruq9oqf271s455d526g3gtev/f2/856bbf0af8a9445cb67ab54889e342c4 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f2/9e38f4621fed4911b0e1dc941ab0c4db_SeqId_9_ 2024-11-24T02:36:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ba713d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:28,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:28,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:28,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:28,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a7ad675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:28,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:28,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,904 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:28,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c940b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:28,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:28,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a78d6fd 2024-11-24T02:36:28,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:28,906 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:28,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:28,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:28,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:28,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:28,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:28,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:28,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:28,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__rbnuc46iihvhfluvnkhulrdf86kv3jj9r9n7c6onruq9oqf271s455d526g3gtev/f2/856bbf0af8a9445cb67ab54889e342c4 into d3c2696984d354e614e3baf276bf25d8/f2 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f2/9e38f4621fed4911b0e1dc941ab0c4db_SeqId_9_ - updating store file list. 2024-11-24T02:36:28,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f2/9e38f4621fed4911b0e1dc941ab0c4db_SeqId_9_ into d3c2696984d354e614e3baf276bf25d8/f2 2024-11-24T02:36:28,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__rbnuc46iihvhfluvnkhulrdf86kv3jj9r9n7c6onruq9oqf271s455d526g3gtev/f2/856bbf0af8a9445cb67ab54889e342c4 into d3c2696984d354e614e3baf276bf25d8/f2 (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/f2/9e38f4621fed4911b0e1dc941ab0c4db_SeqId_9_) 2024-11-24T02:36:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__table1__rbnuc46iihvhfluvnkhulrdf86kv3jj9r9n7c6onruq9oqf271s455d526g3gtev/f2/856bbf0af8a9445cb67ab54889e342c4 2024-11-24T02:36:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f63a4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:28,935 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:28,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:28,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:28,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b9c46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:28,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:28,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,936 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:28,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e752486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:28,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:28,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f1182c 2024-11-24T02:36:28,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:28,938 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T02:36:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:28,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:28,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:28,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T02:36:28,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:28,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:28,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:28,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:28,954 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:28,954 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.incrementalRestoreTable(RestoreTool.java:205) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:185) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:28,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:28,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:28,954 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T02:36:28,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=7 , the old value is region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=7, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:28,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=7 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:28,955 INFO [Time-limited test {}] impl.RestoreTablesClient(187): test-1732415677407 has been successfully restored to table1 2024-11-24T02:36:28,955 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-24T02:36:28,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=7 from cache 2024-11-24T02:36:28,955 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1732415755593 hdfs://localhost:40215/backupUT/backup_1732415755593/default/test-1732415677407/ 2024-11-24T02:36:28,955 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:40215/backupUT/backup_1732415722346/.backup.manifest 2024-11-24T02:36:28,955 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T02:36:28,957 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732415722346/.backup.manifest 2024-11-24T02:36:28,957 INFO [Time-limited test {}] impl.RestoreTablesClient(147): Restoring 'ns2:test-17324156774071' to 'ns2:table2' from full backup image hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071 2024-11-24T02:36:28,958 DEBUG [Time-limited test {}] util.RestoreTool(295): Retrieved descriptor: 'ns2:test-17324156774071', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} thru backup_1732415755593 2024-11-24T02:36:28,961 INFO [Time-limited test {}] util.RestoreTool(480): Truncating exising target table 'ns2:table2', preserving region splits 2024-11-24T02:36:28,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable ns2:table2 2024-11-24T02:36:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=ns2:table2 2024-11-24T02:36:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T02:36:28,965 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415788965"}]},"ts":"1732415788965"} 2024-11-24T02:36:28,966 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2024-11-24T02:36:28,966 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set ns2:table2 to state=DISABLING 2024-11-24T02:36:28,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=ns2:table2}] 2024-11-24T02:36:28,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, UNASSIGN}] 2024-11-24T02:36:28,969 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, UNASSIGN 2024-11-24T02:36:28,970 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=91 updating hbase:meta row=7253aee85dc96326b8dc14f1506394b7, regionState=CLOSING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:28,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, UNASSIGN because future has completed 2024-11-24T02:36:28,971 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:36:28,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7253aee85dc96326b8dc14f1506394b7, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:29,063 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'table1', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 34 more 2024-11-24T02:36:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T02:36:29,124 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(122): Close 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:29,125 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:36:29,125 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1722): Closing 7253aee85dc96326b8dc14f1506394b7, disabling compactions & flushes 2024-11-24T02:36:29,125 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1755): Closing region ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:29,125 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:29,125 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. after waiting 0 ms 2024-11-24T02:36:29,125 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:29,136 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T02:36:29,137 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:29,138 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1973): Closed ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. 2024-11-24T02:36:29,138 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1676): Region close journal for 7253aee85dc96326b8dc14f1506394b7: Waiting for close lock at 1732415789125Running coprocessor pre-close hooks at 1732415789125Disabling compacts and flushes for region at 1732415789125Disabling writes for close at 1732415789125Writing region close event to WAL at 1732415789127 (+2 ms)Running coprocessor post-close hooks at 1732415789137 (+10 ms)Closed at 1732415789138 (+1 ms) 2024-11-24T02:36:29,141 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(157): Closed 7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:29,142 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=91 updating hbase:meta row=7253aee85dc96326b8dc14f1506394b7, regionState=CLOSED 2024-11-24T02:36:29,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7253aee85dc96326b8dc14f1506394b7, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:29,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-24T02:36:29,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; CloseRegionProcedure 7253aee85dc96326b8dc14f1506394b7, server=a67e7d17e593,38807,1732415667927 in 175 msec 2024-11-24T02:36:29,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-24T02:36:29,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=7253aee85dc96326b8dc14f1506394b7, UNASSIGN in 180 msec 2024-11-24T02:36:29,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-24T02:36:29,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=ns2:table2 in 183 msec 2024-11-24T02:36:29,153 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415789153"}]},"ts":"1732415789153"} 2024-11-24T02:36:29,155 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2024-11-24T02:36:29,155 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set ns2:table2 to state=DISABLED 2024-11-24T02:36:29,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; DisableTableProcedure table=ns2:table2 in 194 msec 2024-11-24T02:36:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-24T02:36:29,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: ns2:table2 completed 2024-11-24T02:36:29,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.HMaster$6(2602): Client=jenkins//172.17.0.2 truncate ns2:table2 2024-11-24T02:36:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) 2024-11-24T02:36:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:36:29,287 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:table2' regions in transition 2024-11-24T02:36:29,293 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:29,295 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/f, FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/recovered.edits] 2024-11-24T02:36:29,300 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/f/03b72e0253874be294dd94c699c76d7a_SeqId_4_ to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/archive/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/f/03b72e0253874be294dd94c699c76d7a_SeqId_4_ 2024-11-24T02:36:29,304 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/recovered.edits/6.seqid to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/archive/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7/recovered.edits/6.seqid 2024-11-24T02:36:29,305 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/7253aee85dc96326b8dc14f1506394b7 2024-11-24T02:36:29,305 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived ns2:table2 regions 2024-11-24T02:36:29,310 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2024-11-24T02:36:29,313 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'ns2:table2' descriptor. 2024-11-24T02:36:29,313 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'ns2:table2' from region states. 2024-11-24T02:36:29,314 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732415789313"}]},"ts":"9223372036854775807"} 2024-11-24T02:36:29,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:36:29,316 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7253aee85dc96326b8dc14f1506394b7, NAME => 'ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:36:29,316 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'ns2:table2' as deleted. 2024-11-24T02:36:29,316 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732415789316"}]},"ts":"9223372036854775807"} 2024-11-24T02:36:29,318 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table ns2:table2 state from META 2024-11-24T02:36:29,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741992_1168 (size=351) 2024-11-24T02:36:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:36:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:36:29,730 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 55c1fd40d4d21397b47faf60853b65f1, NAME => 'ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725 2024-11-24T02:36:29,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741993_1169 (size=37) 2024-11-24T02:36:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:36:30,143 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:30,143 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1722): Closing 55c1fd40d4d21397b47faf60853b65f1, disabling compactions & flushes 2024-11-24T02:36:30,143 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:30,143 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:30,143 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. after waiting 0 ms 2024-11-24T02:36:30,143 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:30,143 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1973): Closed ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:30,143 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1676): Region close journal for 55c1fd40d4d21397b47faf60853b65f1: Waiting for close lock at 1732415790143Disabling compacts and flushes for region at 1732415790143Disabling writes for close at 1732415790143Writing region close event to WAL at 1732415790143Closed at 1732415790143 2024-11-24T02:36:30,147 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1732415790146"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732415790146"}]},"ts":"1732415790146"} 2024-11-24T02:36:30,150 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:36:30,151 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415790151"}]},"ts":"1732415790151"} 2024-11-24T02:36:30,154 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2024-11-24T02:36:30,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=55c1fd40d4d21397b47faf60853b65f1, ASSIGN}] 2024-11-24T02:36:30,155 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=55c1fd40d4d21397b47faf60853b65f1, ASSIGN 2024-11-24T02:36:30,157 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=55c1fd40d4d21397b47faf60853b65f1, ASSIGN; state=OFFLINE, location=a67e7d17e593,38807,1732415667927; forceNewPlan=false, retain=false 2024-11-24T02:36:30,308 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=55c1fd40d4d21397b47faf60853b65f1, regionState=OPENING, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:30,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:table2, region=55c1fd40d4d21397b47faf60853b65f1, ASSIGN because future has completed 2024-11-24T02:36:30,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55c1fd40d4d21397b47faf60853b65f1, server=a67e7d17e593,38807,1732415667927}] 2024-11-24T02:36:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:36:30,474 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:30,475 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 55c1fd40d4d21397b47faf60853b65f1, NAME => 'ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:36:30,476 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-24T02:36:30,476 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,476 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:36:30,477 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,477 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,479 INFO [StoreOpener-55c1fd40d4d21397b47faf60853b65f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,480 INFO [StoreOpener-55c1fd40d4d21397b47faf60853b65f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55c1fd40d4d21397b47faf60853b65f1 columnFamilyName f 2024-11-24T02:36:30,480 DEBUG [StoreOpener-55c1fd40d4d21397b47faf60853b65f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:30,481 INFO [StoreOpener-55c1fd40d4d21397b47faf60853b65f1-1 {}] regionserver.HStore(327): Store=55c1fd40d4d21397b47faf60853b65f1/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:36:30,481 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,482 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,482 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,483 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,483 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,484 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,487 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/55c1fd40d4d21397b47faf60853b65f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:36:30,487 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 55c1fd40d4d21397b47faf60853b65f1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11294965280, jitterRate=0.05192561447620392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T02:36:30,487 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:30,488 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 55c1fd40d4d21397b47faf60853b65f1: Running coprocessor pre-open hook at 1732415790477Writing region info on filesystem at 1732415790477Initializing all the Stores at 1732415790478 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732415790478Cleaning up temporary data from old regions at 1732415790483 (+5 ms)Running coprocessor post-open hooks at 1732415790487 (+4 ms)Region opened successfully at 1732415790488 (+1 ms) 2024-11-24T02:36:30,489 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1., pid=95, masterSystemTime=1732415790465 2024-11-24T02:36:30,490 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:30,490 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:30,491 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=55c1fd40d4d21397b47faf60853b65f1, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,38807,1732415667927 2024-11-24T02:36:30,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55c1fd40d4d21397b47faf60853b65f1, server=a67e7d17e593,38807,1732415667927 because future has completed 2024-11-24T02:36:30,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-24T02:36:30,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 55c1fd40d4d21397b47faf60853b65f1, server=a67e7d17e593,38807,1732415667927 in 181 msec 2024-11-24T02:36:30,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-24T02:36:30,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:table2, region=55c1fd40d4d21397b47faf60853b65f1, ASSIGN in 340 msec 2024-11-24T02:36:30,496 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732415790496"}]},"ts":"1732415790496"} 2024-11-24T02:36:30,497 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2024-11-24T02:36:30,497 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:table2' completed 2024-11-24T02:36:30,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; TruncateTableProcedure (table=ns2:table2 preserveSplits=true) in 1.2130 sec 2024-11-24T02:36:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41423 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:36:31,432 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: TRUNCATE, Table Name: ns2:table2 completed 2024-11-24T02:36:31,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T02:36:31,438 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037 from hbase tables ns2:test-17324156774071 to tables ns2:table2 2024-11-24T02:36:31,438 INFO [Time-limited test {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17324156774071 into ns2:table2 2024-11-24T02:36:31,442 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(112): add incremental job :hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415791439 from hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:36:31,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d9ea14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:31,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:31,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:31,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:31,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:31,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:31,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ebd494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:31,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:31,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:31,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:31,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:31,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e8aedc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:31,446 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:31,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:31,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@348a15e2 2024-11-24T02:36:31,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:31,448 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:31,449 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-24T02:36:31,449 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:table2 2024-11-24T02:36:31,449 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T02:36:31,449 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:31,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:31,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:31,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:31,453 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:table2: 2024-11-24T02:36:31,453 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 1 reduce partitions to match current region count for all tables 2024-11-24T02:36:31,453 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/partitions_b4af9b19-077f-4821-b635-ace1814fc31e 2024-11-24T02:36:31,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741994_1170 (size=153) 2024-11-24T02:36:31,518 WARN [regionserver/a67e7d17e593:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 15, running: 1 2024-11-24T02:36:31,536 DEBUG [regionserver/a67e7d17e593:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-24T02:36:31,554 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-24T02:36:31,860 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:31,861 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-protocol-shaded/target/hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:31,861 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-client/target/hbase-client-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,720 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop-5195846137048521620.jar 2024-11-24T02:36:32,720 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-hadoop-compat/target/hbase-hadoop-compat-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics/target/hbase-metrics-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-metrics-api/target/hbase-metrics-api-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-replication/target/hbase-replication-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-http/target/hbase-http-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-procedure/target/hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-zookeeper/target/hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:32,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-24T02:36:32,722 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-24T02:36:32,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-24T02:36:32,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-24T02:36:32,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-24T02:36:32,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-24T02:36:32,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-24T02:36:32,723 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-24T02:36:32,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-24T02:36:32,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-24T02:36:32,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,724 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-common/target/hbase-common-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-mapreduce/target/hbase-mapreduce-4.0.0-alpha-1-SNAPSHOT.jar 2024-11-24T02:36:32,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-24T02:36:32,725 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:table2 2024-11-24T02:36:32,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:32,725 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.createSubmittableJob(MapReduceHFileSplitterJob.java:123) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob.run(MapReduceHFileSplitterJob.java:170) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:82) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:32,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:32,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:32,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:32,726 DEBUG [Time-limited test {}] mapreduce.MapReduceHFileSplitterJob(124): success configuring load incremental job 2024-11-24T02:36:32,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-24T02:36:32,735 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-24T02:36:32,746 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-24T02:36:32,843 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local818969941_0016_338803e5-7ce6-486b-bc4b-371d405914b0/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/*': No such file or directory 2024-11-24T02:36:32,843 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/mapreduce.cluster.local.dir/job_local818969941_0016_338803e5-7ce6-486b-bc4b-371d405914b0/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/libjars/* 2024-11-24T02:36:32,885 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/4a72dce212c94e28ad913a969e06b255 2024-11-24T02:36:32,889 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:32,918 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(92): Initialize HFileRecordReader for hdfs://localhost:40215/backupUT/backup_1732415722346/ns2/test-17324156774071/archive/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/f/8d78355b3323442fa3b7a801ae673cb9 2024-11-24T02:36:32,921 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(106): Seeking to start 2024-11-24T02:36:32,947 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540466b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:32,947 DEBUG [pool-286-thread-1 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:32,948 DEBUG [pool-286-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:32,948 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:32,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:32,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:32,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76df674e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:32,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:32,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:32,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:32,950 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:32,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2cbf34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:32,951 DEBUG [pool-286-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:32,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:32,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:32,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:32,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='row-t20', locateType=CURRENT is [region=ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:32,954 INFO [pool-286-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-286-thread-1. 2024-11-24T02:36:32,954 DEBUG [pool-286-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:32,955 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:32,955 DEBUG [pool-286-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:32,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:32,955 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-24T02:36:32,955 DEBUG [pool-286-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: a67e7d17e593 2024-11-24T02:36:32,959 DEBUG [pool-286-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415791439/_temporary/0/_temporary/attempt_local818969941_0016_r_000000_0/f/a48fbfd939a349549dd14543c98530fa is 38, key is row-t20/f:q1/1732415707124/Put/seqid=0 2024-11-24T02:36:32,960 WARN [Thread-1963 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [a67e7d17e593:38807] Specified favored nodes: [a67e7d17e593:38807] 2024-11-24T02:36:32,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741995_1171 (size=8934) 2024-11-24T02:36:33,884 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(87): Restoring HFiles from directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415791439 2024-11-24T02:36:33,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4593774a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:33,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:33,886 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:33,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:33,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:33,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7cc3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:33,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:33,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,888 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:33,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d562861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,889 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:33,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:33,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:33,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:33,893 WARN [Time-limited test {}] tool.BulkLoadHFilesTool(259): Skipping non-directory hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415791439/_SUCCESS 2024-11-24T02:36:33,896 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:33,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:33,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6cebe8e7 2024-11-24T02:36:33,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:33,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:33,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:33,902 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:table2,, stopping at row=ns2:table2 ,, for max=2147483647 with caching=100 2024-11-24T02:36:33,907 INFO [BulkLoadHFilesTool-0 {}] tool.BulkLoadHFilesTool(704): Trying to load hfile=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415791439/f/a48fbfd939a349549dd14543c98530fa first=Optional[row-t20] last=Optional[row98] 2024-11-24T02:36:33,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415791439/f/a48fbfd939a349549dd14543c98530fa for inclusion in 55c1fd40d4d21397b47faf60853b65f1/f 2024-11-24T02:36:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(626): HFile bounds: first=row-t20 last=row98 2024-11-24T02:36:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T02:36:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegion(2603): Flush status journal for 55c1fd40d4d21397b47faf60853b65f1: 2024-11-24T02:36:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/hbase-staging/bulk_output-ns2-table2-1732415791439/f/a48fbfd939a349549dd14543c98530fa to hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__o6eda20as7iiuq977s3u6154o1kk940299hnnobfql1vpil9f5fcj0qvgqo8n5s6/f/a48fbfd939a349549dd14543c98530fa 2024-11-24T02:36:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__o6eda20as7iiuq977s3u6154o1kk940299hnnobfql1vpil9f5fcj0qvgqo8n5s6/f/a48fbfd939a349549dd14543c98530fa as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/55c1fd40d4d21397b47faf60853b65f1/f/3cc29edabb1641289f85d294ab89941a_SeqId_4_ 2024-11-24T02:36:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b48bb8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:33,918 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:33,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:33,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:33,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f6b3b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:33,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:33,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,920 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:33,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2042a505, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:33,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:33,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7832727f 2024-11-24T02:36:33,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:33,922 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:36:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:33,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:33,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:33,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:36:33,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:33,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:33,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:33,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.preCommitStoreFile(BackupObserver.java:105) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1447) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$51.call(RegionCoprocessorHost.java:1444) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCommitStoreFile(RegionCoprocessorHost.java:1443) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:7401) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:288) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager$1.run(SecureBulkLoadManager.java:263) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:263) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:33,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__o6eda20as7iiuq977s3u6154o1kk940299hnnobfql1vpil9f5fcj0qvgqo8n5s6/f/a48fbfd939a349549dd14543c98530fa into 55c1fd40d4d21397b47faf60853b65f1/f as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/55c1fd40d4d21397b47faf60853b65f1/f/3cc29edabb1641289f85d294ab89941a_SeqId_4_ - updating store file list. 2024-11-24T02:36:33,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/55c1fd40d4d21397b47faf60853b65f1/f/3cc29edabb1641289f85d294ab89941a_SeqId_4_ into 55c1fd40d4d21397b47faf60853b65f1/f 2024-11-24T02:36:33,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__o6eda20as7iiuq977s3u6154o1kk940299hnnobfql1vpil9f5fcj0qvgqo8n5s6/f/a48fbfd939a349549dd14543c98530fa into 55c1fd40d4d21397b47faf60853b65f1/f (new location: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/55c1fd40d4d21397b47faf60853b65f1/f/3cc29edabb1641289f85d294ab89941a_SeqId_4_) 2024-11-24T02:36:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/staging/jenkins__ns2_table2__o6eda20as7iiuq977s3u6154o1kk940299hnnobfql1vpil9f5fcj0qvgqo8n5s6/f/a48fbfd939a349549dd14543c98530fa 2024-11-24T02:36:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65396e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,41423,-1 for getting cluster id 2024-11-24T02:36:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:36:33,946 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec8b847e-5725-4935-9e6f-e811370a6839' 2024-11-24T02:36:33,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:36:33,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec8b847e-5725-4935-9e6f-e811370a6839" 2024-11-24T02:36:33,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@83b919f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,41423,-1] 2024-11-24T02:36:33,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:36:33,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,948 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:36:33,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1874ba60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-24T02:36:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:36:33,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,41423,1732415666135 2024-11-24T02:36:33,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d6aa906 2024-11-24T02:36:33,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:36:33,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-24T02:36:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:36:33,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,38807,1732415667927, seqNum=-1] 2024-11-24T02:36:33,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:36:33,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-24T02:36:33,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-24T02:36:33,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-24T02:36:33,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:', locateType=CURRENT is [region=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:33,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807. 2024-11-24T02:36:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupObserver.postBulkLoadHFile(BackupObserver.java:79) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1474) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$53.call(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postBulkLoadHFile(RegionCoprocessorHost.java:1471) at org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.secureBulkLoadHFiles(SecureBulkLoadManager.java:307) at org.apache.hadoop.hbase.regionserver.RSRpcServices.bulkLoadHFile(RSRpcServices.java:2347) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43510) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:33,967 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:33,967 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.tool.BulkLoadHFilesTool.bulkLoad(BulkLoadHFilesTool.java:1125) at org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreJob.run(MapReduceRestoreJob.java:90) at org.apache.hadoop.hbase.backup.util.RestoreTool.createAndRestoreTable(RestoreTool.java:355) at org.apache.hadoop.hbase.backup.util.RestoreTool.fullRestoreTable(RestoreTool.java:211) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restoreImages(RestoreTablesClient.java:150) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.restore(RestoreTablesClient.java:226) at org.apache.hadoop.hbase.backup.impl.RestoreTablesClient.execute(RestoreTablesClient.java:260) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.restore(BackupAdminImpl.java:509) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:213) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:33,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:33,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:33,968 DEBUG [Time-limited test {}] mapreduce.MapReduceRestoreJob(95): Restore Job finished:0 2024-11-24T02:36:33,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 , the old value is region=ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1., hostname=a67e7d17e593,38807,1732415667927, seqNum=2, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:33,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-11-24T02:36:33,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 from cache 2024-11-24T02:36:33,968 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T02:36:33,984 INFO [Time-limited test {}] impl.RestoreTablesClient(175): No incremental changes since full backup for 'ns2:test-17324156774071', skipping incremental restore step. 2024-11-24T02:36:33,984 INFO [Time-limited test {}] impl.RestoreTablesClient(229): Restore includes the following image(s): 2024-11-24T02:36:33,984 INFO [Time-limited test {}] impl.RestoreTablesClient(231): Backup: backup_1732415755593 hdfs://localhost:40215/backupUT/backup_1732415755593/ns2/test-17324156774071/ 2024-11-24T02:36:33,984 DEBUG [Time-limited test {}] impl.RestoreTablesClient(236): restoreStage finished 2024-11-24T02:36:33,985 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(217): After incremental restore: 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:36:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.CallRunner(138): callId: 740 service: ClientService methodName: Scan size: 160 connection: 172.17.0.2:40306 deadline: 1732415853985, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. is not online on a67e7d17e593,38807,1732415667927 2024-11-24T02:36:33,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 , the old value is region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. is not online on a67e7d17e593,38807,1732415667927 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:33,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,,1732415768837.c8065a920e0f1af7868a104494e4732d. is not online on a67e7d17e593,38807,1732415667927 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:33,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,,1732415768837.c8065a920e0f1af7868a104494e4732d., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 from cache 2024-11-24T02:36:34,072 WARN [Async-Client-Retry-Timer-pool-0 {}] client.AsyncNonMetaRegionLocator(265): Failed to locate region in 'ns2:table2', row='', locateType=CURRENT org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=a67e7d17e593:38807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.scan(ClientProtos.java:43851) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.callOpenScanner(AsyncClientScanner.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.call(AsyncRpcRetryingCaller.java:222) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory$SingleRequestCallerBuilder.call(AsyncRpcRetryingCallerFactory.java:177) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:242) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.timelineConsistentRead(ConnectionUtils.java:442) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.openScanner(AsyncClientScanner.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClientScanner.start(AsyncClientScanner.java:275) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:617) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.scan(RawAsyncTableImpl.java:91) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.locateInMeta(AsyncNonMetaRegionLocator.java:408) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocationsInternal(AsyncNonMetaRegionLocator.java:516) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncNonMetaRegionLocator.getRegionLocations(AsyncNonMetaRegionLocator.java:529) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.lambda$getRegionLocation$7(AsyncRegionLocator.java:164) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.tracedLocationFuture(AsyncRegionLocator.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:158) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionLocator.getRegionLocation(AsyncRegionLocator.java:193) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 34 more 2024-11-24T02:36:34,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='', locateType=CURRENT is [region=table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8., hostname=a67e7d17e593,38807,1732415667927, seqNum=7] 2024-11-24T02:36:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.CallRunner(138): callId: 744 service: ClientService methodName: Scan size: 170 connection: 172.17.0.2:40306 deadline: 1732415854103, exception=org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. is not online on a67e7d17e593,38807,1732415667927 2024-11-24T02:36:34,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 , the old value is region=table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271., hostname=a67e7d17e593,38807,1732415667927, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. is not online on a67e7d17e593,38807,1732415667927 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:34,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271. is not online on a67e7d17e593,38807,1732415667927 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:34,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=table1,row0,1732415768837.1523f0656aca8ceaf15f6bcfbf77d271., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 from cache 2024-11-24T02:36:34,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row0', locateType=CURRENT is [region=table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c., hostname=a67e7d17e593,38807,1732415667927, seqNum=7] 2024-11-24T02:36:34,221 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(219): f1 has 198 rows 2024-11-24T02:36:34,228 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(223): f2 has 7 rows 2024-11-24T02:36:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:34,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T02:36:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T02:36:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T02:36:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:34,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T02:36:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:34,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T02:36:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:36:34,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] regionserver.HMobStore(347): The Cell result is null, assemble a new Cell with the same row,family,qualifier,timestamp,type and tags but with an empty value to return. 2024-11-24T02:36:34,255 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(227): mob has 111 rows 2024-11-24T02:36:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38807 {}] ipc.CallRunner(138): callId: 752 service: ClientService methodName: Scan size: 159 connection: 172.17.0.2:40306 deadline: 1732415854254, exception=org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. is not online on a67e7d17e593,38807,1732415667927 2024-11-24T02:36:34,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 , the old value is region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. is not online on a67e7d17e593,38807,1732415667927 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:34,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7. is not online on a67e7d17e593,38807,1732415667927 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:36:34,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7., hostname=a67e7d17e593,38807,1732415667927, seqNum=2 from cache 2024-11-24T02:36:34,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:table2', row='', locateType=CURRENT is [region=ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1., hostname=a67e7d17e593,38807,1732415667927, seqNum=2] 2024-11-24T02:36:34,367 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:34,368 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:235) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:34,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:34,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:34,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:34,388 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=635 (was 611) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: PacketResponder: BP-1481479676-172.17.0.2-1732415661183:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: 'a67e7d17e593,38807,1732415667927' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/a67e7d17e593:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-94687300_20 at /127.0.0.1:54694 [Waiting for operation #115] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1481479676-172.17.0.2-1732415661183:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1481479676-172.17.0.2-1732415661183:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1138450493_20 at /127.0.0.1:34988 [Receiving block BP-1481479676-172.17.0.2-1732415661183:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1138450493_20 at /127.0.0.1:34992 [Receiving block BP-1481479676-172.17.0.2-1732415661183:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(a67e7d17e593,38807,1732415667927)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (a67e7d17e593,41423,1732415666135)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1138450493_20 at /127.0.0.1:34980 [Receiving block BP-1481479676-172.17.0.2-1732415661183:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=643 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=97 (was 217), ProcessCount=17 (was 17), AvailableMemoryMB=5274 (was 5791) 2024-11-24T02:36:34,388 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=635 is superior to 500 2024-11-24T02:36:34,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T02:36:34,395 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:34,395 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:34,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:34,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:34,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:34,396 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T02:36:34,396 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2009745068, stopped=false 2024-11-24T02:36:34,396 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a67e7d17e593,41423,1732415666135 2024-11-24T02:36:34,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-24T02:36:34,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-24T02:36:34,443 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T02:36:34,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:36:34,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:36:34,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-24T02:36:34,444 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:36:34,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-24T02:36:34,444 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:34,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:34,445 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a67e7d17e593,38807,1732415667927' ***** 2024-11-24T02:36:34,445 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T02:36:34,445 INFO [RS:0;a67e7d17e593:38807 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-24T02:36:34,448 INFO [RS:0;a67e7d17e593:38807 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@72979367{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-24T02:36:34,450 INFO [RS:0;a67e7d17e593:38807 {}] server.AbstractConnector(383): Stopped ServerConnector@1cdf8417{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-24T02:36:34,450 INFO [RS:0;a67e7d17e593:38807 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:36:34,451 INFO [RS:0;a67e7d17e593:38807 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@160a6036{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-24T02:36:34,451 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T02:36:34,451 INFO [RS:0;a67e7d17e593:38807 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@65eb8eac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,STOPPED} 2024-11-24T02:36:34,452 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 48740e06c1a2c00320ec32fe075fe704 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 55c1fd40d4d21397b47faf60853b65f1 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for a9fcfc65d96498f86ae4291b18ffd315 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 50af115aeed7c572d01a2d7d6548f037 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 6b1451007b9af7358fedf38311721e6c 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a605441c1cb1d296fb15e4b88f05f9c 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 6df51c7a7a1591e81897afbbac0da0ea 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(959): stopping server a67e7d17e593,38807,1732415667927 2024-11-24T02:36:34,453 INFO [RS:0;a67e7d17e593:38807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T02:36:34,454 INFO [RS:0;a67e7d17e593:38807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a67e7d17e593:38807. 2024-11-24T02:36:34,454 DEBUG [RS:0;a67e7d17e593:38807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:34,454 DEBUG [RS:0;a67e7d17e593:38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:34,454 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 48740e06c1a2c00320ec32fe075fe704, disabling compactions & flushes 2024-11-24T02:36:34,454 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:36:34,454 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T02:36:34,454 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:36:34,454 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T02:36:34,454 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T02:36:34,454 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. after waiting 0 ms 2024-11-24T02:36:34,454 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:36:34,454 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T02:36:34,454 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-24T02:36:34,454 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T02:36:34,454 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T02:36:34,454 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T02:36:34,454 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T02:36:34,454 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T02:36:34,455 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a605441c1cb1d296fb15e4b88f05f9c, 48740e06c1a2c00320ec32fe075fe704, 50af115aeed7c572d01a2d7d6548f037, 55c1fd40d4d21397b47faf60853b65f1, 6b1451007b9af7358fedf38311721e6c, 6df51c7a7a1591e81897afbbac0da0ea, a9fcfc65d96498f86ae4291b18ffd315, aca5da70eca26b7decb49ebc97e25da8, d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:34,455 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=23.97 KB heapSize=42.40 KB 2024-11-24T02:36:34,458 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system_bulk/48740e06c1a2c00320ec32fe075fe704/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T02:36:34,458 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:34,458 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:36:34,458 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 48740e06c1a2c00320ec32fe075fe704: Waiting for close lock at 1732415794453Running coprocessor pre-close hooks at 1732415794454 (+1 ms)Disabling compacts and flushes for region at 1732415794454Disabling writes for close at 1732415794454Writing region close event to WAL at 1732415794454Running coprocessor post-close hooks at 1732415794458 (+4 ms)Closed at 1732415794458 2024-11-24T02:36:34,458 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1732415688839.48740e06c1a2c00320ec32fe075fe704. 2024-11-24T02:36:34,459 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 55c1fd40d4d21397b47faf60853b65f1, disabling compactions & flushes 2024-11-24T02:36:34,459 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:34,459 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:34,459 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. after waiting 0 ms 2024-11-24T02:36:34,459 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:34,463 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/table2/55c1fd40d4d21397b47faf60853b65f1/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2024-11-24T02:36:34,463 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:34,463 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:34,463 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 55c1fd40d4d21397b47faf60853b65f1: Waiting for close lock at 1732415794459Running coprocessor pre-close hooks at 1732415794459Disabling compacts and flushes for region at 1732415794459Disabling writes for close at 1732415794459Writing region close event to WAL at 1732415794459Running coprocessor post-close hooks at 1732415794463 (+4 ms)Closed at 1732415794463 2024-11-24T02:36:34,463 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:table2,,1732415789306.55c1fd40d4d21397b47faf60853b65f1. 2024-11-24T02:36:34,464 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a9fcfc65d96498f86ae4291b18ffd315, disabling compactions & flushes 2024-11-24T02:36:34,464 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:36:34,464 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:36:34,464 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. after waiting 0 ms 2024-11-24T02:36:34,464 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:36:34,464 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a9fcfc65d96498f86ae4291b18ffd315 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-24T02:36:34,476 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/info/4c532084f14945ffa589095131d2f62a is 159, key is ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037./info:regioninfo/1732415681038/Put/seqid=0 2024-11-24T02:36:34,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741996_1172 (size=19093) 2024-11-24T02:36:34,483 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f2/50e3b4ff5264442d8fee2444cabd5909 is 58, key is row-test-1732415677407-2-0/f2:q1/1732415750583/Put/seqid=0 2024-11-24T02:36:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741997_1173 (size=5464) 2024-11-24T02:36:34,514 INFO [regionserver/a67e7d17e593:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T02:36:34,514 INFO [regionserver/a67e7d17e593:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T02:36:34,544 INFO [regionserver/a67e7d17e593:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T02:36:34,655 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a605441c1cb1d296fb15e4b88f05f9c, 50af115aeed7c572d01a2d7d6548f037, 6b1451007b9af7358fedf38311721e6c, 6df51c7a7a1591e81897afbbac0da0ea, a9fcfc65d96498f86ae4291b18ffd315, aca5da70eca26b7decb49ebc97e25da8, d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:34,855 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a605441c1cb1d296fb15e4b88f05f9c, 50af115aeed7c572d01a2d7d6548f037, 6b1451007b9af7358fedf38311721e6c, 6df51c7a7a1591e81897afbbac0da0ea, a9fcfc65d96498f86ae4291b18ffd315, aca5da70eca26b7decb49ebc97e25da8, d3c2696984d354e614e3baf276bf25d8 2024-11-24T02:36:34,882 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.86 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/info/4c532084f14945ffa589095131d2f62a 2024-11-24T02:36:34,888 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f2/50e3b4ff5264442d8fee2444cabd5909 2024-11-24T02:36:34,898 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/.tmp/f2/50e3b4ff5264442d8fee2444cabd5909 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f2/50e3b4ff5264442d8fee2444cabd5909 2024-11-24T02:36:34,903 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/f2/50e3b4ff5264442d8fee2444cabd5909, entries=7, sequenceid=348, filesize=5.3 K 2024-11-24T02:36:34,904 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for a9fcfc65d96498f86ae4291b18ffd315 in 440ms, sequenceid=348, compaction requested=false 2024-11-24T02:36:34,909 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/test-1732415677407/a9fcfc65d96498f86ae4291b18ffd315/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-24T02:36:34,909 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:34,909 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:36:34,909 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a9fcfc65d96498f86ae4291b18ffd315: Waiting for close lock at 1732415794464Running coprocessor pre-close hooks at 1732415794464Disabling compacts and flushes for region at 1732415794464Disabling writes for close at 1732415794464Obtaining lock to block concurrent updates at 1732415794464Preparing flush snapshotting stores in a9fcfc65d96498f86ae4291b18ffd315 at 1732415794464Finished memstore snapshotting test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1732415794464Flushing stores of test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. at 1732415794465 (+1 ms)Flushing a9fcfc65d96498f86ae4291b18ffd315/f2: creating writer at 1732415794465Flushing a9fcfc65d96498f86ae4291b18ffd315/f2: appending metadata at 1732415794482 (+17 ms)Flushing a9fcfc65d96498f86ae4291b18ffd315/f2: closing flushed file at 1732415794482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e8e5ef4: reopening flushed file at 1732415794897 (+415 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for a9fcfc65d96498f86ae4291b18ffd315 in 440ms, sequenceid=348, compaction requested=false at 1732415794904 (+7 ms)Writing region close event to WAL at 1732415794906 (+2 ms)Running coprocessor post-close hooks at 1732415794909 (+3 ms)Closed at 1732415794909 2024-11-24T02:36:34,909 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1732415677407,,1732415677907.a9fcfc65d96498f86ae4291b18ffd315. 2024-11-24T02:36:34,909 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 50af115aeed7c572d01a2d7d6548f037, disabling compactions & flushes 2024-11-24T02:36:34,909 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:36:34,909 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:36:34,910 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. after waiting 0 ms 2024-11-24T02:36:34,910 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:36:34,913 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns2/test-17324156774071/50af115aeed7c572d01a2d7d6548f037/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-24T02:36:34,913 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:34,913 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:36:34,914 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 50af115aeed7c572d01a2d7d6548f037: Waiting for close lock at 1732415794909Running coprocessor pre-close hooks at 1732415794909Disabling compacts and flushes for region at 1732415794909Disabling writes for close at 1732415794910 (+1 ms)Writing region close event to WAL at 1732415794910Running coprocessor post-close hooks at 1732415794913 (+3 ms)Closed at 1732415794913 2024-11-24T02:36:34,914 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17324156774071,,1732415680259.50af115aeed7c572d01a2d7d6548f037. 2024-11-24T02:36:34,914 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6b1451007b9af7358fedf38311721e6c, disabling compactions & flushes 2024-11-24T02:36:34,914 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:34,914 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:34,914 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. after waiting 0 ms 2024-11-24T02:36:34,914 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:34,917 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/6b1451007b9af7358fedf38311721e6c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=6 2024-11-24T02:36:34,918 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/ns/6f2e650f9318478d8f42c411979da80f is 85, key is ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7./ns:/1732415789308/DeleteFamily/seqid=0 2024-11-24T02:36:34,918 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:34,918 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:34,918 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6b1451007b9af7358fedf38311721e6c: Waiting for close lock at 1732415794914Running coprocessor pre-close hooks at 1732415794914Disabling compacts and flushes for region at 1732415794914Disabling writes for close at 1732415794914Writing region close event to WAL at 1732415794914Running coprocessor post-close hooks at 1732415794918 (+4 ms)Closed at 1732415794918 2024-11-24T02:36:34,918 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,row0,1732415779711.6b1451007b9af7358fedf38311721e6c. 2024-11-24T02:36:34,918 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a605441c1cb1d296fb15e4b88f05f9c, disabling compactions & flushes 2024-11-24T02:36:34,918 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:36:34,918 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:36:34,918 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. after waiting 0 ms 2024-11-24T02:36:34,918 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:36:34,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741998_1174 (size=5760) 2024-11-24T02:36:34,922 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns4/test-17324156774073/1a605441c1cb1d296fb15e4b88f05f9c/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T02:36:34,922 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:34,922 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:36:34,922 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a605441c1cb1d296fb15e4b88f05f9c: Waiting for close lock at 1732415794918Running coprocessor pre-close hooks at 1732415794918Disabling compacts and flushes for region at 1732415794918Disabling writes for close at 1732415794918Writing region close event to WAL at 1732415794919 (+1 ms)Running coprocessor post-close hooks at 1732415794922 (+3 ms)Closed at 1732415794922 2024-11-24T02:36:34,922 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17324156774073,,1732415683722.1a605441c1cb1d296fb15e4b88f05f9c. 2024-11-24T02:36:34,923 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d3c2696984d354e614e3baf276bf25d8, disabling compactions & flushes 2024-11-24T02:36:34,923 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:34,923 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:34,923 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. after waiting 0 ms 2024-11-24T02:36:34,923 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:34,926 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/default/table1/d3c2696984d354e614e3baf276bf25d8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=6 2024-11-24T02:36:34,927 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:34,927 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:34,927 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d3c2696984d354e614e3baf276bf25d8: Waiting for close lock at 1732415794923Running coprocessor pre-close hooks at 1732415794923Disabling compacts and flushes for region at 1732415794923Disabling writes for close at 1732415794923Writing region close event to WAL at 1732415794924 (+1 ms)Running coprocessor post-close hooks at 1732415794927 (+3 ms)Closed at 1732415794927 2024-11-24T02:36:34,927 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1732415779711.d3c2696984d354e614e3baf276bf25d8. 2024-11-24T02:36:34,927 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6df51c7a7a1591e81897afbbac0da0ea, disabling compactions & flushes 2024-11-24T02:36:34,927 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:36:34,927 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:36:34,927 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. after waiting 0 ms 2024-11-24T02:36:34,927 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:36:34,927 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6df51c7a7a1591e81897afbbac0da0ea 2/2 column families, dataSize=2.12 KB heapSize=3.43 KB 2024-11-24T02:36:34,930 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/bb3761000b8b454b8c9652fba09be6f9 is 158, key is trslm:hdfs://localhost:40215/backupUT\x00ns2:test-17324156774071/meta:log-roll-map/1732415768337/Put/seqid=0 2024-11-24T02:36:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741999_1175 (size=6208) 2024-11-24T02:36:35,055 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6df51c7a7a1591e81897afbbac0da0ea, aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:36:35,256 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6df51c7a7a1591e81897afbbac0da0ea, aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:36:35,322 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=474 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/ns/6f2e650f9318478d8f42c411979da80f 2024-11-24T02:36:35,334 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=944 B at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/bb3761000b8b454b8c9652fba09be6f9 2024-11-24T02:36:35,344 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/642613980ccd452794de7ed80263a863 is 588, key is session:backup_1732415755593/session:context/1732415768778/Put/seqid=0 2024-11-24T02:36:35,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073742000_1176 (size=5689) 2024-11-24T02:36:35,350 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/rep_barrier/0fbc8cc55b114cb49ef9787d63da6552 is 94, key is ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7./rep_barrier:/1732415789308/DeleteFamily/seqid=0 2024-11-24T02:36:35,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073742001_1177 (size=5640) 2024-11-24T02:36:35,456 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-24T02:36:35,456 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6df51c7a7a1591e81897afbbac0da0ea=backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., aca5da70eca26b7decb49ebc97e25da8=ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8.} 2024-11-24T02:36:35,456 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6df51c7a7a1591e81897afbbac0da0ea, aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:36:35,657 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6df51c7a7a1591e81897afbbac0da0ea, aca5da70eca26b7decb49ebc97e25da8 2024-11-24T02:36:35,749 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/642613980ccd452794de7ed80263a863 2024-11-24T02:36:35,755 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=266 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/rep_barrier/0fbc8cc55b114cb49ef9787d63da6552 2024-11-24T02:36:35,761 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/meta/bb3761000b8b454b8c9652fba09be6f9 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/bb3761000b8b454b8c9652fba09be6f9 2024-11-24T02:36:35,768 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/meta/bb3761000b8b454b8c9652fba09be6f9, entries=7, sequenceid=52, filesize=6.1 K 2024-11-24T02:36:35,769 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/.tmp/session/642613980ccd452794de7ed80263a863 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/642613980ccd452794de7ed80263a863 2024-11-24T02:36:35,775 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/session/642613980ccd452794de7ed80263a863, entries=2, sequenceid=52, filesize=5.6 K 2024-11-24T02:36:35,775 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for 6df51c7a7a1591e81897afbbac0da0ea in 848ms, sequenceid=52, compaction requested=true 2024-11-24T02:36:35,780 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/backup/system/6df51c7a7a1591e81897afbbac0da0ea/recovered.edits/55.seqid, newMaxSeqId=55, maxSeqId=1 2024-11-24T02:36:35,780 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:35,780 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:36:35,780 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6df51c7a7a1591e81897afbbac0da0ea: Waiting for close lock at 1732415794927Running coprocessor pre-close hooks at 1732415794927Disabling compacts and flushes for region at 1732415794927Disabling writes for close at 1732415794927Obtaining lock to block concurrent updates at 1732415794927Preparing flush snapshotting stores in 6df51c7a7a1591e81897afbbac0da0ea at 1732415794927Finished memstore snapshotting backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea., syncing WAL and waiting on mvcc, flushsize=dataSize=2170, getHeapSize=3480, getOffHeapSize=0, getCellsCount=11 at 1732415794927Flushing stores of backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. at 1732415794928 (+1 ms)Flushing 6df51c7a7a1591e81897afbbac0da0ea/meta: creating writer at 1732415794928Flushing 6df51c7a7a1591e81897afbbac0da0ea/meta: appending metadata at 1732415794930 (+2 ms)Flushing 6df51c7a7a1591e81897afbbac0da0ea/meta: closing flushed file at 1732415794930Flushing 6df51c7a7a1591e81897afbbac0da0ea/session: creating writer at 1732415795341 (+411 ms)Flushing 6df51c7a7a1591e81897afbbac0da0ea/session: appending metadata at 1732415795344 (+3 ms)Flushing 6df51c7a7a1591e81897afbbac0da0ea/session: closing flushed file at 1732415795344Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34278a1b: reopening flushed file at 1732415795760 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cb720e4: reopening flushed file at 1732415795769 (+9 ms)Finished flush of dataSize ~2.12 KB/2170, heapSize ~3.40 KB/3480, currentSize=0 B/0 for 6df51c7a7a1591e81897afbbac0da0ea in 848ms, sequenceid=52, compaction requested=true at 1732415795775 (+6 ms)Writing region close event to WAL at 1732415795777 (+2 ms)Running coprocessor post-close hooks at 1732415795780 (+3 ms)Closed at 1732415795780 2024-11-24T02:36:35,781 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1732415687677.6df51c7a7a1591e81897afbbac0da0ea. 2024-11-24T02:36:35,781 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aca5da70eca26b7decb49ebc97e25da8, disabling compactions & flushes 2024-11-24T02:36:35,781 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:36:35,781 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:36:35,781 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. after waiting 0 ms 2024-11-24T02:36:35,781 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:36:35,781 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/table/8d3395c58e3f4e8b9f99fa10cc7db423 is 88, key is ns2:table2,,1732415774633.7253aee85dc96326b8dc14f1506394b7./table:/1732415789308/DeleteFamily/seqid=0 2024-11-24T02:36:35,785 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/ns3/test-17324156774072/aca5da70eca26b7decb49ebc97e25da8/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T02:36:35,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073742002_1178 (size=6364) 2024-11-24T02:36:35,785 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:35,785 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:36:35,785 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aca5da70eca26b7decb49ebc97e25da8: Waiting for close lock at 1732415795781Running coprocessor pre-close hooks at 1732415795781Disabling compacts and flushes for region at 1732415795781Disabling writes for close at 1732415795781Writing region close event to WAL at 1732415795781Running coprocessor post-close hooks at 1732415795785 (+4 ms)Closed at 1732415795785 2024-11-24T02:36:35,785 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17324156774072,,1732415681557.aca5da70eca26b7decb49ebc97e25da8. 2024-11-24T02:36:35,857 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:36:36,057 DEBUG [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:36:36,187 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.39 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/table/8d3395c58e3f4e8b9f99fa10cc7db423 2024-11-24T02:36:36,203 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/info/4c532084f14945ffa589095131d2f62a as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/info/4c532084f14945ffa589095131d2f62a 2024-11-24T02:36:36,208 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/info/4c532084f14945ffa589095131d2f62a, entries=114, sequenceid=97, filesize=18.6 K 2024-11-24T02:36:36,209 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/ns/6f2e650f9318478d8f42c411979da80f as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/ns/6f2e650f9318478d8f42c411979da80f 2024-11-24T02:36:36,215 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/ns/6f2e650f9318478d8f42c411979da80f, entries=10, sequenceid=97, filesize=5.6 K 2024-11-24T02:36:36,216 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/rep_barrier/0fbc8cc55b114cb49ef9787d63da6552 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/rep_barrier/0fbc8cc55b114cb49ef9787d63da6552 2024-11-24T02:36:36,222 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/rep_barrier/0fbc8cc55b114cb49ef9787d63da6552, entries=3, sequenceid=97, filesize=5.5 K 2024-11-24T02:36:36,223 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/.tmp/table/8d3395c58e3f4e8b9f99fa10cc7db423 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/table/8d3395c58e3f4e8b9f99fa10cc7db423 2024-11-24T02:36:36,227 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/table/8d3395c58e3f4e8b9f99fa10cc7db423, entries=21, sequenceid=97, filesize=6.2 K 2024-11-24T02:36:36,228 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1774ms, sequenceid=97, compaction requested=false 2024-11-24T02:36:36,232 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/data/hbase/meta/1588230740/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2024-11-24T02:36:36,233 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:36,233 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T02:36:36,233 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T02:36:36,233 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732415794454Running coprocessor pre-close hooks at 1732415794454Disabling compacts and flushes for region at 1732415794454Disabling writes for close at 1732415794454Obtaining lock to block concurrent updates at 1732415794455 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732415794455Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=24544, getHeapSize=43352, getOffHeapSize=0, getCellsCount=235 at 1732415794455Flushing stores of hbase:meta,,1.1588230740 at 1732415794455Flushing 1588230740/info: creating writer at 1732415794455Flushing 1588230740/info: appending metadata at 1732415794476 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732415794476Flushing 1588230740/ns: creating writer at 1732415794893 (+417 ms)Flushing 1588230740/ns: appending metadata at 1732415794917 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1732415794917Flushing 1588230740/rep_barrier: creating writer at 1732415795329 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732415795350 (+21 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732415795350Flushing 1588230740/table: creating writer at 1732415795763 (+413 ms)Flushing 1588230740/table: appending metadata at 1732415795781 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732415795781Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d18ce50: reopening flushed file at 1732415796202 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f5849fc: reopening flushed file at 1732415796209 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44b6d1d2: reopening flushed file at 1732415796216 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d968515: reopening flushed file at 1732415796222 (+6 ms)Finished flush of dataSize ~23.97 KB/24544, heapSize ~42.34 KB/43352, currentSize=0 B/0 for 1588230740 in 1774ms, sequenceid=97, compaction requested=false at 1732415796228 (+6 ms)Writing region close event to WAL at 1732415796229 (+1 ms)Running coprocessor post-close hooks at 1732415796233 (+4 ms)Closed at 1732415796233 2024-11-24T02:36:36,233 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T02:36:36,257 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(976): stopping server a67e7d17e593,38807,1732415667927; all regions closed. 2024-11-24T02:36:36,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741955_1131 (size=16160) 2024-11-24T02:36:36,261 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.meta.1732415760062.meta not finished, retry = 0 2024-11-24T02:36:36,374 DEBUG [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs 2024-11-24T02:36:36,374 INFO [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.meta:.meta(num 1732415760062) 2024-11-24T02:36:36,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741956_1132 (size=7926) 2024-11-24T02:36:36,377 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1.1732415760077 not finished, retry = 0 2024-11-24T02:36:36,487 DEBUG [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs 2024-11-24T02:36:36,487 INFO [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-1:(num 1732415760077) 2024-11-24T02:36:36,491 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/WALs/a67e7d17e593,38807,1732415667927/a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0.1732415760044 not finished, retry = 0 2024-11-24T02:36:36,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741954_1130 (size=4122) 2024-11-24T02:36:36,598 DEBUG [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/oldWALs 2024-11-24T02:36:36,598 INFO [RS:0;a67e7d17e593:38807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a67e7d17e593%2C38807%2C1732415667927.a67e7d17e593%2C38807%2C1732415667927.regiongroup-0:(num 1732415760044) 2024-11-24T02:36:36,598 DEBUG [RS:0;a67e7d17e593:38807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:36,598 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T02:36:36,598 INFO [RS:0;a67e7d17e593:38807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T02:36:36,599 INFO [RS:0;a67e7d17e593:38807 {}] hbase.ChoreService(370): Chore service for: regionserver/a67e7d17e593:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T02:36:36,600 INFO [RS:0;a67e7d17e593:38807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T02:36:36,600 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T02:36:36,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17324156774072 2024-11-24T02:36:36,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T02:36:36,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-24T02:36:36,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2024-11-24T02:36:36,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T02:36:36,602 INFO [RS:0;a67e7d17e593:38807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38807 2024-11-24T02:36:36,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-24T02:36:36,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17324156774073 2024-11-24T02:36:36,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-24T02:36:36,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732415677407 2024-11-24T02:36:36,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17324156774071 2024-11-24T02:36:36,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2024-11-24T02:36:36,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/a67e7d17e593,38807,1732415667927 2024-11-24T02:36:36,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-24T02:36:36,651 INFO [RS:0;a67e7d17e593:38807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T02:36:36,661 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a67e7d17e593,38807,1732415667927] 2024-11-24T02:36:36,667 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/a67e7d17e593,38807,1732415667927 already deleted, retry=false 2024-11-24T02:36:36,667 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a67e7d17e593,38807,1732415667927 expired; onlineServers=0 2024-11-24T02:36:36,668 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a67e7d17e593,41423,1732415666135' ***** 2024-11-24T02:36:36,668 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T02:36:36,668 INFO [M:0;a67e7d17e593:41423 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-24T02:36:36,673 INFO [M:0;a67e7d17e593:41423 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@6b147c85{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/master} 2024-11-24T02:36:36,674 INFO [M:0;a67e7d17e593:41423 {}] server.AbstractConnector(383): Stopped ServerConnector@3bf75574{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-24T02:36:36,674 INFO [M:0;a67e7d17e593:41423 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:36:36,674 INFO [M:0;a67e7d17e593:41423 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@e90fde3{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-24T02:36:36,674 INFO [M:0;a67e7d17e593:41423 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@613be4cc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,STOPPED} 2024-11-24T02:36:36,674 INFO [M:0;a67e7d17e593:41423 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T02:36:36,674 INFO [M:0;a67e7d17e593:41423 {}] master.HMaster(1770): Stopping master jetty server 2024-11-24T02:36:36,675 INFO [M:0;a67e7d17e593:41423 {}] server.AbstractConnector(383): Stopped ServerConnector@3007dbc3{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-24T02:36:36,675 INFO [M:0;a67e7d17e593:41423 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T02:36:36,675 DEBUG [M:0;a67e7d17e593:41423 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T02:36:36,675 INFO [M:0;a67e7d17e593:41423 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-24T02:36:36,675 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T02:36:36,676 DEBUG [master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.large.0-1732415671282 {}] cleaner.HFileCleaner(306): Exit Thread[master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.large.0-1732415671282,5,FailOnTimeoutGroup] 2024-11-24T02:36:36,676 DEBUG [master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.small.0-1732415671287 {}] cleaner.HFileCleaner(306): Exit Thread[master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.small.0-1732415671287,5,FailOnTimeoutGroup] 2024-11-24T02:36:36,676 INFO [M:0;a67e7d17e593:41423 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;a67e7d17e593:41423. 2024-11-24T02:36:36,676 DEBUG [M:0;a67e7d17e593:41423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:36:36,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-24T02:36:36,676 DEBUG [M:0;a67e7d17e593:41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:36,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-24T02:36:36,676 DEBUG [M:0;a67e7d17e593:41423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:36:36,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:36:36,676 DEBUG [M:0;a67e7d17e593:41423 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T02:36:36,676 INFO [M:0;a67e7d17e593:41423 {}] hbase.ChoreService(370): Chore service for: master/a67e7d17e593:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T02:36:36,676 INFO [M:0;a67e7d17e593:41423 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T02:36:36,676 DEBUG [M:0;a67e7d17e593:41423 {}] master.HMaster(1795): Stopping service threads 2024-11-24T02:36:36,676 INFO [M:0;a67e7d17e593:41423 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T02:36:36,677 INFO [M:0;a67e7d17e593:41423 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T02:36:36,677 ERROR [M:0;a67e7d17e593:41423 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-1,5,PEWorkerGroup] Thread[HFileArchiver-2,5,PEWorkerGroup] Thread[HFileArchiver-3,5,PEWorkerGroup] 2024-11-24T02:36:36,677 INFO [M:0;a67e7d17e593:41423 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T02:36:36,677 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T02:36:36,678 DEBUG [M:0;a67e7d17e593:41423 {}] zookeeper.ZKUtil(347): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-24T02:36:36,678 WARN [M:0;a67e7d17e593:41423 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T02:36:36,678 INFO [M:0;a67e7d17e593:41423 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/.lastflushedseqids 2024-11-24T02:36:36,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073742003_1179 (size=761) 2024-11-24T02:36:36,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:36:36,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38807-0x1016bfac7200001, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:36:36,761 INFO [RS:0;a67e7d17e593:38807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T02:36:36,761 INFO [RS:0;a67e7d17e593:38807 {}] regionserver.HRegionServer(1031): Exiting; stopping=a67e7d17e593,38807,1732415667927; zookeeper connection closed. 2024-11-24T02:36:36,762 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a26258e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a26258e 2024-11-24T02:36:36,762 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T02:36:37,090 INFO [M:0;a67e7d17e593:41423 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T02:36:37,090 INFO [M:0;a67e7d17e593:41423 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T02:36:37,091 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T02:36:37,107 INFO [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:36:37,107 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:36:37,107 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T02:36:37,108 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:36:37,108 INFO [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=441.11 KB heapSize=518.09 KB 2024-11-24T02:36:37,122 DEBUG [M:0;a67e7d17e593:41423 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f73c0c545c0442e39861b68d0b521f82 is 82, key is hbase:meta,,1/info:regioninfo/1732415672620/Put/seqid=0 2024-11-24T02:36:37,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073742004_1180 (size=5672) 2024-11-24T02:36:37,528 INFO [M:0;a67e7d17e593:41423 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f73c0c545c0442e39861b68d0b521f82 2024-11-24T02:36:37,555 DEBUG [M:0;a67e7d17e593:41423 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d43f844167e24863bbda69eef7a40efc is 2201, key is \x00\x00\x00\x00\x00\x00\x00Q/proc:d/1732415786442/Put/seqid=0 2024-11-24T02:36:37,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073742005_1181 (size=30143) 2024-11-24T02:36:37,960 INFO [M:0;a67e7d17e593:41423 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=440.55 KB at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d43f844167e24863bbda69eef7a40efc 2024-11-24T02:36:37,970 INFO [M:0;a67e7d17e593:41423 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d43f844167e24863bbda69eef7a40efc 2024-11-24T02:36:37,986 DEBUG [M:0;a67e7d17e593:41423 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/984767a7ef8941779268fc63236a434b is 69, key is a67e7d17e593,38807,1732415667927/rs:state/1732415671399/Put/seqid=0 2024-11-24T02:36:37,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073742006_1182 (size=5156) 2024-11-24T02:36:38,392 INFO [M:0;a67e7d17e593:41423 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=908 (bloomFilter=true), to=hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/984767a7ef8941779268fc63236a434b 2024-11-24T02:36:38,403 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f73c0c545c0442e39861b68d0b521f82 as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f73c0c545c0442e39861b68d0b521f82 2024-11-24T02:36:38,408 INFO [M:0;a67e7d17e593:41423 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f73c0c545c0442e39861b68d0b521f82, entries=8, sequenceid=908, filesize=5.5 K 2024-11-24T02:36:38,409 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d43f844167e24863bbda69eef7a40efc as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d43f844167e24863bbda69eef7a40efc 2024-11-24T02:36:38,413 INFO [M:0;a67e7d17e593:41423 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d43f844167e24863bbda69eef7a40efc 2024-11-24T02:36:38,413 INFO [M:0;a67e7d17e593:41423 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d43f844167e24863bbda69eef7a40efc, entries=94, sequenceid=908, filesize=29.4 K 2024-11-24T02:36:38,414 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/984767a7ef8941779268fc63236a434b as hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/984767a7ef8941779268fc63236a434b 2024-11-24T02:36:38,418 INFO [M:0;a67e7d17e593:41423 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40215/user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/984767a7ef8941779268fc63236a434b, entries=1, sequenceid=908, filesize=5.0 K 2024-11-24T02:36:38,419 INFO [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(3140): Finished flush of dataSize ~441.11 KB/451697, heapSize ~517.79 KB/530216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1311ms, sequenceid=908, compaction requested=false 2024-11-24T02:36:38,420 INFO [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:36:38,420 DEBUG [M:0;a67e7d17e593:41423 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732415797091Disabling compacts and flushes for region at 1732415797091Disabling writes for close at 1732415797107 (+16 ms)Obtaining lock to block concurrent updates at 1732415797108 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732415797108Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=451697, getHeapSize=530456, getOffHeapSize=0, getCellsCount=1031 at 1732415797108Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732415797108Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732415797108Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732415797121 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732415797121Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732415797538 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732415797554 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732415797554Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732415797970 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732415797985 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732415797985Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c3d4392: reopening flushed file at 1732415798402 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4abcbb33: reopening flushed file at 1732415798408 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56e51dab: reopening flushed file at 1732415798414 (+6 ms)Finished flush of dataSize ~441.11 KB/451697, heapSize ~517.79 KB/530216, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1311ms, sequenceid=908, compaction requested=false at 1732415798419 (+5 ms)Writing region close event to WAL at 1732415798420 (+1 ms)Closed at 1732415798420 2024-11-24T02:36:38,423 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c5cbe4f5-51f7-b035-677f-aa3381b32725/MasterData/WALs/a67e7d17e593,41423,1732415666135/a67e7d17e593%2C41423%2C1732415666135.a67e7d17e593%2C41423%2C1732415666135.regiongroup-0.1732415670211 not finished, retry = 0 2024-11-24T02:36:38,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34653 is added to blk_1073741830_1006 (size=511628) 2024-11-24T02:36:38,526 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T02:36:38,526 INFO [M:0;a67e7d17e593:41423 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T02:36:38,527 INFO [M:0;a67e7d17e593:41423 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-24T02:36:38,527 INFO [M:0;a67e7d17e593:41423 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41423 2024-11-24T02:36:38,527 INFO [M:0;a67e7d17e593:41423 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T02:36:38,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:36:38,684 INFO [M:0;a67e7d17e593:41423 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T02:36:38,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41423-0x1016bfac7200000, quorum=127.0.0.1:58752, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:36:38,697 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6628e1d0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T02:36:38,700 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d5d982f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:36:38,700 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:36:38,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6be2f62b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T02:36:38,701 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b9b7476{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,STOPPED} 2024-11-24T02:36:38,704 WARN [BP-1481479676-172.17.0.2-1732415661183 heartbeating to localhost/127.0.0.1:40215 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T02:36:38,704 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T02:36:38,704 WARN [BP-1481479676-172.17.0.2-1732415661183 heartbeating to localhost/127.0.0.1:40215 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1481479676-172.17.0.2-1732415661183 (Datanode Uuid fba3894c-5686-4c5f-8aec-1ea8dccd98b4) service to localhost/127.0.0.1:40215 2024-11-24T02:36:38,704 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T02:36:38,706 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/data/data1/current/BP-1481479676-172.17.0.2-1732415661183 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T02:36:38,707 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/cluster_8135818e-0777-9b52-9ae1-7ae344601417/data/data2/current/BP-1481479676-172.17.0.2-1732415661183 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T02:36:38,707 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T02:36:38,713 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6df1a11{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T02:36:38,713 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34c8bc74{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:36:38,714 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:36:38,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17280962{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T02:36:38,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71f46724{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,STOPPED} 2024-11-24T02:36:38,722 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T02:36:38,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T02:36:38,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-24T02:36:38,788 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a47ddee{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-24T02:36:38,788 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2bf04c0f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:36:38,789 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:36:38,789 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17558f00{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-24T02:36:38,789 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ce56c02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,STOPPED} 2024-11-24T02:36:38,800 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7ac3fdb2{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-24T02:36:38,801 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@14fd9c4e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:36:38,801 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:36:38,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38b35098{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-24T02:36:38,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13757798{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,STOPPED} 2024-11-24T02:36:38,805 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-24T02:36:38,805 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@33bd426e{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-24T02:36:38,805 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@423a9db4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:36:38,806 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:36:38,806 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6f34157b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-24T02:36:38,806 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@349f1fd6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,STOPPED} 2024-11-24T02:36:38,808 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-24T02:36:38,811 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-24T02:36:38,811 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-24T02:36:38,811 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-24T02:36:38,814 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5018e614{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-24T02:36:38,814 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@66767106{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:36:38,815 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:36:38,815 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22311523{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-24T02:36:38,815 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32dc3d85{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-backup/target/test-data/e7a9cc42-7f6f-6d7a-d781-6d8e7c28cf0e/hadoop.log.dir/,STOPPED} 2024-11-24T02:36:38,817 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-24T02:36:38,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped